from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from random_user_agent.user_agent import UserAgent

from random_user_agent.params import SoftwareName, OperatingSystem
import openpyxl
import random
import time

search_char = ['L','S','T']

software_names = [SoftwareName.CHROME.value]
operating_systems = [OperatingSystem.WINDOWS.value,
                     OperatingSystem.LINUX.value]
user_agent_rotator = UserAgent(software_names = software_names,
                       operating_systems = operating_systems,
                       limit=100)
user_agent = user_agent_rotator.get_random_user_agent()

def click_element(elem_type=0, elem_selector=0):
    if(elem_type == 'css'):
        elem = browser.find_element_by_css_selector(elem_selector)
    elem.click

def sleep_fuc():
    time.sleep(random.randrange(2,4))

def get_search_window(srch_address,srch_prim_plan,srch_sub_plan,row_num,col_num,active_sheet):
    global browser
    global software_names, operating_systems, user_agent_rotator, user_agent

    chrome_options = Options()
    chrome_options.add_argument("user-agent ={}".format(user_agent))

    browser = webdriver.Chrome(options=chrome_options)
    browser.get('https://www.blueshieldca.com/fap/app/find-a-doctor.html')

    browser.switch_to.window(browser.window_handles[1])

    doctor_button=browser.find_element_by_id('Doctor')

    doctor_button.click()

    #guest_button=find_element_by_id('nonMember')
    WebDriverWait(browser,2).until(EC.element_to_be_clickable((By.ID,'nonMember'))).click()

    #guest_button.click()

    location_input=browser.find_element_by_id('autocomplete')

    #location_input.click()
    #location_input.send_keys('Los Angeles, CA 90002, USA')

    sleep_fuc()

    while(browser.current_url != 'https://www.blueshieldca.com/fad/plans/chooseplan'):
        location_input.clear()
        location_input.send_keys(srch_address)

        location_input.send_keys(Keys.LEFT)
        location_input.send_keys(Keys.RIGHT)

        action = webdriver.common.action_chains.ActionChains(browser)
        action.move_to_element_with_offset(location_input, 100,60)
        action.click()
        action.perform()

        sleep_fuc()

        #CLICK CONTINUE
        #click_element('css','#main > div > app-location > div.location-container > div > div.wrapper-location.col-sm-12.col-md-10.col-lg-7 > button')
        continue_button = browser.find_element_by_css_selector('#main > div > app-location > div.location-container > div > div.wrapper-location.col-sm-12.col-md-10.col-lg-7 > button')
        sleep_fuc()

        continue_button.click()

    sleep_fuc()
    for tries in range(0,5):
        try:
            browser.refresh()
            allHTML = browser.find_element_by_tag_name('html')
            allHTML.send_keys(Keys.PAGE_DOWN)
            time.sleep(1)
            primary_plan_button = browser.find_element_by_css_selector('#main > div > app-choose-plans > div > div > div > div > div > button.btn.btn-primary.planButtonLeft')
            primary_plan_button.click()

            #CLICK PRIMARY PLAN DROPDOWN
            #click_element('css','#primaryPlanDropdown > div')
            #htmlElem = browser.find_element_by_tag_name('html')
            #htmlElem.send_keys(Keys.PAGE_DOWN)
            primary_plan_dropdown = browser.find_element_by_xpath('//*[@id="primaryPlanDropdown"]')
            browser.execute_script("arguments[0].click();", primary_plan_dropdown)
            time.sleep(2)

            primary_plan_ul = browser.find_element_by_xpath('//*[@id="main"]/div/app-plans/div[1]/div/div/div/form/div/div/div/div[3]/div[2]/ul')
            primary_plans = primary_plan_ul.find_elements_by_tag_name('li')

            srch_prim_plan = srch_prim_plan.replace(" ","").lower()
            for plan in primary_plans:
                plan_text = plan.text
                plan_text = plan_text.replace(" ","").lower()
                if (plan_text == srch_prim_plan):
                    plan.click()
                    break

            try:
                sub_plan_dropdown = browser.find_element_by_xpath('//*[@id="subPlanDropdown"]')
                browser.execute_script("arguments[0].click();", sub_plan_dropdown)

                allHTML = browser.find_element_by_tag_name('html')
                allHTML.send_keys(Keys.PAGE_DOWN)
                time.sleep(1)

                sub_plan_ul = browser.find_element_by_xpath('//*[@id="main"]/div/app-plans/div[1]/div/div/div/form/div/div/div/div[3]/div[3]/ul')
                sub_plans = sub_plan_ul.find_elements_by_tag_name('li')

                #Create search pattern

                srch_sub_plan = srch_sub_plan.replace(" ","")

                for sub in sub_plans:
                    sub_text = sub.text
                    sub_text = sub_text.replace(" ","")

                    if (sub_text == srch_sub_plan):
                        sub.click()
                        break

            except:
                print('No Subplan')

            sleep_fuc()
            plan_continue = browser.find_element_by_xpath('//*[@id="continuePlan"]')
            browser.execute_script("arguments[0].click();", plan_continue)
            sleep_fuc()
            return True
        except Exception as e:
            #print(e)
            print('Request Denied, try# '+str(tries))
            #browser.quit()
            #software_names = [SoftwareName.CHROME.value]
            #operating_systems = [OperatingSystem.WINDOWS.value,
                                 #OperatingSystem.LINUX.value]
            #user_agent_rotator = UserAgent(software_names = software_names,
                                   #operating_systems = operating_systems,
                                   #limit=100)
            #user_agent = user_agent_rotator.get_random_user_agent()
            browser.back()
            time.sleep(2*tries)
            #get_search_window(srch_address,srch_prim_plan,srch_sub_plan,row_num,col_num,active_sheet)
            #active_sheet.cell(row = row_num, column = col_num).value = "0 (Manually Check)"
    return False
#####################

def get_doctor_count(row_num,col_num,active_sheet):
    global search_char
    #doctor_name = browser.find_element_by_xpath('//*[@id="Enter last name"]/label')
    #doctor_name.click()

    WebDriverWait(browser,2).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="Enter last name"]/label'))).click()
    time.sleep(0.3)

    enter_name = WebDriverWait(browser,2).until(EC.presence_of_element_located((By.XPATH,'//*[@id="provider_name"]')))
    enter_name.clear()
    enter_name.send_keys(search_char[0])

    search_btn = browser.find_element_by_xpath('//*[@id="searchBtn"]')
    search_btn.click()

    sleep_fuc()

    try:
        miles = browser.find_element_by_xpath('//*[@id="searchResultMsg"]/a')
        browser.execute_script("arguments[0].click();", miles)
    except:
        total_doctors = "0 (Failed at miles)"
        active_sheet.cell(row = row_num, column = col_num).value = total_doctors
        print('Failed at miles')
        return

    miles_dropdown = browser.find_element_by_xpath('//*[@id="dropdownRadiusFilter"]')
    browser.execute_script("arguments[0].click();", miles_dropdown)

    time.sleep(1.5)

    #miles_ul = browser.find_element_by_xpath('//*[@id="affix_el"]/app-filtersort/div/div[2]/form/fieldset/div[1]/div[1]/div/ul')
    miles_100 = browser.find_element_by_xpath('//*[@id="select_lo7"]')
    miles_100.click()

    apply_btn = browser.find_element_by_xpath('//*[@id="filter-button-apply"]')
    apply_btn.click()
    time.sleep(4)

    try:
        total_doctors = browser.find_element_by_xpath('//*[@id="searchResultCount"]/span[1]')
    except:
        total_doctors = "0"

    active_sheet.cell(row = row_num, column = col_num).value = total_doctors.text
    print('Success!: '+ str(row_num)+', '+str(col_num))

    for index in range(1,3):
        col_num = col_num + 1
        enter_name = WebDriverWait(browser,2).until(EC.presence_of_element_located((By.XPATH,'//*[@id="provider_name"]')))
        enter_name.clear()
        enter_name.send_keys(search_char[index])

        WebDriverWait(browser,2).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="searchBtn"]'))).click()
        time.sleep(5)
        try:
            total_doctors = browser.find_element_by_xpath('//*[@id="searchResultCount"]/span[1]')
        except:
            total_doctors = "0"

        active_sheet.cell(row = row_num, column = col_num).value = total_doctors.text
        print('Success!: '+ str(row_num)+', '+str(col_num))

####################

def main():
    #OPEN EXCEL FILE
    #print('Please input the whole file name. (must be .xlsx file)')
    file = 'Site 5 Product Research Jun21.xlsx'
    wb = openpyxl.load_workbook(file)

    print('Start on which row?')
    start_row = input()
    print('Start on which column?')
    start_column = input()

    sheet = wb['Sheet1']

    for row in range(start_row, sheet.max_row +1):
        sub_plan = sheet['D' + str(row)].value
        prim_plan = sheet['E' + str(row)].value

        for col in range(start_column, sheet.max_column + 1, 3):
            search_value = sheet.cell(row = 1, column = col).value
            zip_code = search_value[0:5]
            #search_char = search_value[-1]

            sheet = wb['Sheet2']
            for zip_row in range(1,sheet.max_row+1):
                if (str(zip_code) == str(sheet['A'+str(zip_row)].value)):
                    address = sheet['B'+str(zip_row)].value
                    break

            sheet = wb['Sheet1']
            proceed = get_search_window(address,prim_plan,sub_plan,row,col,sheet)
            if (proceed):
                get_doctor_count(row,col,sheet)
            wb.save(file)
            browser.quit()

        start_column = 7

    print('Done!')

main()


# reads stories and classifies them as content discussion
def read_stories(stories_path, label_encoder, group_mapper, grouping):
    X = []
    y = []
    cd_code = label_encoder.transform([group_mapper["content discussion"] if grouping else "content discussion"])[0]

    for story_path in stories_path:
        story = open(story_path, 'r')
        lines = story.readlines()

        for line in lines:
            X.append(line.replace("\n", ""))
            y.append(cd_code)

    return X, y


"""Ollie and Tim's super awesome UDP socket thing! OATSAUST

    DATE:   24th July 2016
    DUE:    9th September 2016
    WORTH:  20%
"""

# import socketserver
import socket  # for sockets
import sys # for exit
import struct

def server():
    """Trying to create a socket."""
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # must be dgram because udp
    port = int(sys.argv[1])
    s.bind(("localhost", port))
    while True:
        data, address = s.recvfrom(1024)
        if data.decode() == "request-time":
            print("request-time was received")
            reply = str.encode("kevin")
            s.sendto(reply, address) # check if I must hardcode this
            s.close()
            break

server()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 10 10:49:28 2021

@author: t1
"""

import time
import numpy as np
from collections import Counter

def time_it(func):
    def wrapper(*args,**kwargs):
        start = time.clock()
        result = func(*args,**kwargs)
        end = time.clock()
        print(' {} executed in time : {:.6f} sec'.format(func.__name__,end - start))
        return result
    return wrapper

def read_file(file_path):
    M = None
    N = None
    spectrum = None
    with open(file_path) as f:
        lines = f.readlines()
        M = int(lines[0].strip())
        N = int(lines[1].strip())
        spectrum = [int(x) for x in lines[2].strip().split(' ')]
    return M,N,sorted(spectrum)

def get_convolution2(spectrum):
    '''
    spectrum -> list of ints

    returns :
        conv_list -> list of ints; convolution of spectrum
    '''
    spec_arr = np.array(spectrum)
    conv_mat = spec_arr.reshape(-1,1) - spec_arr.reshape(1,-1)
    conv_list = list(conv_mat[conv_mat > 0].ravel())
    return conv_list

def get_top_M_weights(M,conv_list):
    '''
    M -> int
    conv_list -> list of ints ;

    returns -> top M most common values in convolution
    '''
    conv_list = [el for el in conv_list if (el >= 57) and (el <= 200)]
    counts_ = Counter(conv_list)
    mc_list = counts_.most_common(M)
    last_count_val = mc_list[-1][1]
    for (key,val) in counts_.items():
        if (val == last_count_val) and ((key,val) not in mc_list):
            mc_list.append((key,val))
    return [w for (w,c) in mc_list]

def expand_cands(cand_list,mass_list):
    '''
    cand_list -> list of strs

    returns new collection containing all possible extensions of
    peptides in Peptides by a single amino acid mass
    '''
    new_cand_list = []
    for cand in cand_list:
        for mass in mass_list:
            if cand == "":
                new_cand = str(mass)
            else:
                new_cand = cand + '-' + str(mass)
            new_cand_list.append(new_cand)
    return new_cand_list

def get_cand_mass(cand):
    '''
    cand -> str
    returns total mass for candidate sequence
    '''
    mass_list = [int(m) for m in cand.split('-')]
    return sum(mass_list)

def get_cand_spectrum(cand):

    '''
    cand -> str of masses eg : '113-129-128-114'
    returns cyclo spectrum of peptides
    '''
    mass_list = [int(m) for m in cand.split('-')]
    spectrum = []
    for mass in mass_list:
        spectrum.append(mass)
    for k in range(2,len(mass_list)):
        for i in range(len(mass_list)):
            j = i+k
            w_ = None
            if j > len(mass_list):
                w_ = sum(mass_list[i:]) + sum(mass_list[:(j - len(mass_list))])
            else:
                w_ = sum(mass_list[i:j])
            spectrum.append(w_)
    if len(mass_list) > 1:
        spectrum.append(sum(mass_list))
    spectrum = [0] + spectrum
    return sorted(spectrum)

def get_cand_lin_spectrum(cand):
    mass_list = [int(m) for m in cand.split('-')]
    spectrum = []
    for mass in mass_list:
        spectrum.append(mass)
    for k in range(2,len(mass_list)):
        for i in range(len(mass_list) -k +1):
            j = i+k
            w_ = sum(mass_list[i:j])
            spectrum.append(w_)
    if len(mass_list) > 1:
        spectrum.append(sum(mass_list))
    spectrum = [0] + spectrum
    return sorted(spectrum)

def get_cand_lin_score(peptide,spectrum):
    '''
    peptide -> str
    spectrum -> list of ints
    score is defined as the number of masses shared between
    Cyclospectrum(Peptide) and Spectrum
    '''
    peptide_spectrum = get_cand_lin_spectrum(peptide)
    pep_spectrum_set = list(set(peptide_spectrum))
    score = 0
    for mass in pep_spectrum_set:
        count_1 = peptide_spectrum.count(mass)
        count_2 = spectrum.count(mass)
        score += min(count_1,count_2)
    return score

def get_cand_cyclic_score(peptide,spectrum):
    '''
    peptide -> str
    spectrum -> list of ints
    score is defined as the number of masses shared between
    Cyclospectrum(Peptide) and Spectrum
    '''
    peptide_spectrum = get_cand_spectrum(peptide)
    pep_spectrum_set = list(set(peptide_spectrum))
    score = 0
    for mass in pep_spectrum_set:
        count_1 = peptide_spectrum.count(mass)
        count_2 = spectrum.count(mass)
        score += min(count_1,count_2)
    return score

def trim(N,lead_board,spectrum):
    if len(lead_board) == 0:
        return []
    score_list = [(cand,get_cand_lin_score(cand, spectrum)) for cand in lead_board]
    sorted_score_list = sorted(score_list,key = lambda x : x[1],reverse = True)
    # print(sorted_score_list)
    trimmed_list = sorted_score_list[:N]
    nt_score = trimmed_list[-1][1]
    for cand,score in sorted_score_list[N:]:
        if score == nt_score:
            trimmed_list.append((cand,score))
    trimmed_cand_list = [x[0] for x in trimmed_list]
    # print(len(trimmed_cand_list))
    return trimmed_cand_list

def get_cand_peptide(N,spectrum,mass_list):
    '''
    N -> leaderboard size
    spectrum -> experimental spectrum
    returns peptide cancdidate with the highest score
    '''

    lead_board = [""]
    lead_board_trim = []
    lead_cand = ''
    lead_score = 0
    cand_score_dict = {}
    PARENT_MASS = max(spectrum)
    while len(lead_board) > 0:
        lead_board = expand_cands(lead_board,mass_list)
        lead_board_trim = []

        for cand in lead_board:
            cand_mass = get_cand_mass(cand)
            # cand_score = get_cand_cyclic_score(cand, spectrum)
            # cand_score_dict[cand] = cand_score
            if cand_mass == PARENT_MASS:
                cand_score = get_cand_cyclic_score(cand, spectrum)
                cand_score_dict[cand] = cand_score

                if cand_score > lead_score:
                    lead_cand = cand
                    lead_score = cand_score
            if cand_mass <= PARENT_MASS:
                lead_board_trim.append(cand)
        lead_board = trim(N,lead_board_trim,spectrum)

    return lead_cand

@time_it
def get_conv_cyclo_pep_seq(M,N,spectrum):
    '''
    M -> int ; top M most frequent weight values are taken from convolution of spectrum
    N -> int ; trim top N values in leaderboard cyclopeptide sequencing
    spectrum -> list of ints ; experimental spectrum

    returns : most probable sequence of peptide
    '''
    # get spectrum convolution
    conv_list = get_convolution2(spectrum)
    mass_list = get_top_M_weights(M, conv_list)
    # print('masses : ',len(mass_list),mass_list)

    lead_cand = get_cand_peptide(N, spectrum, mass_list)
    return lead_cand

# file_path = 'sample_data/convol_cyclo_pep_seq.txt'
file_path = 'test_data/dataset_104_7.txt'
M,N,spectrum = read_file(file_path)
lead_cand = get_conv_cyclo_pep_seq(M, N, spectrum)
print(lead_cand)


TARIFF_11 = 0.244618
TARIFF_31 = 0.136928

def temp_convert():
    """
    CP1404/CP5632 - Practical
    Pseudocode for temperature conversion
    """

    MENU = """    C - Convert Celsius to Fahrenheit
    F - Convert Fahrenheit to Celsius
    Q - Quit"""
    print(MENU)
    choice = input(">>> ").upper()
    while choice != "Q":
        if choice == "C":
            celsius = float(input("Celsius: "))
            fahrenheit = celsius * 9.0 / 5 + 32
            print("Result: {:.2f} F".format(fahrenheit))
        elif choice == "F":
            # TODO: Write this section to convert F to C and display the result
            # Hint: celsius = 5 / 9 * (fahrenheit - 32)
            # Remove the "pass" statement when you are done. It's a placeholder.
            # pass
            fahrenheit = float(input("Fahrenheit: "))
            celsius = 5 / 9 * (fahrenheit - 32)
            print("Result: {:.2f} C".format(celsius))
        else:
            print("Invalid option")
        print(MENU)
        choice = input(">>> ").upper()
    print("Thank you.")

def sales_bonus():
    """
    Program to calculate and display a user's bonus based on sales.
    If sales are under $1,000, the user gets a 10% bonus.
    If sales are $1,000 or over, the bonus is 15%.
    """
    sales = float(input("Sales Amount:$ "))
    if sales < 0:
        print("Invalid Amount")
    elif sales < 1000:
        bonus = sales / 100 * 10
        print("Bonus:$ {:.2f}".format(bonus))
    else:
        bonus = sales / 100 * 15
        print("Bonus:$ {:.2f}".format(bonus))

def grades():
    score = float(input("Enter score: "))
    if score < 0 or score > 100:
        print("Invalid score")
    elif score > 90:
        print("Excellent")
    elif score >= 50:
        print("Passable")
    else:
        print("Bad")

def loops():
    for i in range(1, 21, 2):
        print(i, end=' ')

    print("\n")

    for i in range(0, 101, 10):
        print(i, end=' ')

    print("\n")

    for i in range(20, 0, -1):
        print(i, end=' ')
    # print()

def sales_bonus_loop():
    """
    Program to calculate and display a user's bonus based on sales.
    If sales are under $1,000, the user gets a 10% bonus.
    If sales are $1,000 or over, the bonus is 15%.
    """
    sales = 0
    while sales >= 0:
        sales = float(input("Sales Amount:$ "))
        if sales < 0:
            print("Invalid Amount")
        elif sales < 1000:
            bonus = sales / 100 * 10
            print("Bonus:$ {:.2f}".format(bonus))
        else:
            bonus = sales / 100 * 15
            print("Bonus:$ {:.2f}".format(bonus))

def shop_calculator():
    items = int(input("Number of items: "))
    while items < 0:
        print("Invalid number of items!")
        items = int(input("Number of items: "))

    prices = [0] * items
    total_price = 0
    for i in range(0, items, 1):
        prices[i] = float(input("Price of item: "))
        while prices[i] < 0:
            print("Invalid price!")
            prices[i] = float(input("Price of item: "))

        total_price += prices[i]

    if total_price >= 100:
        # total_price = total_price / 10 * 9
        total_price *= 0.9
        print("Total price for", items, "item(s) is ${:.2f}".format(total_price))
    else:
        print("Total price for", items, "item(s) is ${:.2f}".format(total_price))

def electric_bill():
    print("Electricity bill estimator\n")
    bill = float(input("Enter cents per kWh: "))
    electricity = float(input("Enter daily use in kWh: "))
    total_days = float(input("Enter number of billing days: "))
    estimate_bill = (bill * electricity * total_days) / 100

    print("Estimated bill: $ {:.2f}".format(estimate_bill))

def electric_bill_2():
    print("Electricity bill estimator 2\n")
    tarif_choice = float(input("Which tariff? 11 or 31: "))
    if tarif_choice == 11:
        tarif_selection = TARIFF_11
    elif tarif_choice == 31:
        tarif_selection = TARIFF_31

    electricity = float(input("Enter daily use in kWh: "))
    total_days = float(input("Enter number of billing days: "))
    estimate_bill = (tarif_selection * electricity * total_days)

    print("Estimated bill: $ {:.2f}".format(estimate_bill))

def main():
    # print("Hello World")
    # Part 1
    temp_convert()

    # Part 2
    # sales_bonus()

    # Part 3
    # grades()

    # Part 4
    # loops()

    # Part 5
    # sales_bonus_loop()

    # Part 6
    # shop_calculator()

    # Part 7
    #electric_bill()

    # Part 8
    # electric_bill_2()

main()


#!/usr/bin/env python3
import os, os.path
import time

from . import *

debug("Loading modules...")
from .converters import *	# harnesses for (external) converters
from .m3u import *			# VLC extended m3u file produced by Clipper.lua
from .cutlist import *		# intermediate Windows format
from .splits_tsv import *	# user-defined tab-separated file

###
def get_converters(*args, **kwargs):
	if not args:
		return [], kwargs
	video_file = cuts = cut_units = '' # wouldn't do that with a classier object
	for arg in args:
		_, ext = os.path.splitext(arg)
		ext = ext.upper()
		if '.M3U' == ext:
			cuts, cut_units = [ (cut.start, cut.end) for cut in extended_m3u_file(arg) ], 'seconds' # decimal
		elif '.CUTLIST' == ext:
			cuts, cut_units = [ (cut.start, cut.end) for cut in cutlist(arg).cuts ], 'seconds' # decimal
		elif '.SPLITS' == ext:
			cuts, cut_units = old_splits_file(arg).cuts, 'frames'
		elif ext in ('.GIF', '.OGV'):
			video_file, converters = arg, [ffmpeg]
		elif ext in ('.AVI', '.DIVX'):
			video_file, converters = arg, [avidemux, ffmpeg]
		elif ext in ('.MKV', '.WEBM', '.FLV'):
			video_file, converters = arg, [mkvmerge, ffmpeg, avidemux]
		elif ext in ('.MPG', '.MP4', '.M4V', '.MOV', '.F4V', '.3GP', '.3G2', '.MJ2'):
			video_file, converters = arg, [MP4Box, mkvmerge, ffmpeg]
		elif ext in ('.ASF', '.WMV'):
			video_file, converters = arg, [asfbin, ffmpeg]
		else:
			raise SplitterException("{} not supported".format(arg))
		if any(f in kwargs for f in [ 'filters', 'audio_filters', 'video_filters' ] ):
			[ converters.remove(c) for c in [asfbin, mkvmerge, MP4Box] if c in converters ]
	if 'converters' in kwargs:
		info("Replacing default converters {}".format(converters))
		converter = kwargs.pop('converters')
	nargs = {}
	# can be overwritten within kwargs:
	nargs['video_file'] = video_file
	# this section defines 'seconds' as the default 'splits'
	if 'seconds' == cut_units:
		nargs['splits'] = cuts
	else:
		nargs[cut_units] = cuts
	#
	nargs.update(kwargs) # what's left is passed through
	# ... so these are never overridden by kwargs:
	#nargs['profile'] = converters[0].__name__
	#
	return converters, nargs

def main(*args, **kwargs):
	debug("Arguments:")
	for arg in args:
		debug("\t{}".format(arg))
	converters, kwargs = get_converters(*args, **kwargs)
	if 'converter' in kwargs:
		converters = [kwargs.pop('converter')]
	elif 'converters' in kwargs:
		converters = kwargs.pop('converters')
	if not converters:
		panic("No files in {} supported".format(args))
		return -1
	video_file = kwargs.pop('video_file')
	size = os.path.getsize(video_file)
	debug("{} is {:,} B".format(video_file, size))
	debug("Parameters:")
	for k, v in kwargs.items():
		debug("\t{}={}".format(k, v))
	st, dur = time.time(), 0.
	for is_retry, c in enumerate(converters):
		name, options = c.__name__, dict(kwargs) # kwargs can be modified by converter methods
		if not is_retry:
			info("Trying {}...".format(name))
		else:
			warning("Retry {} ({})...".format(is_retry, name))
			options['ext'] = '.MKV'
		st = time.time() # reset
		try:
			rc = c(video_file, dry_run=options['--dry-run'], **options)
		except OSError as e:
			panic("{} not found: {}".format(name, e))
			return -2
		except SplitterException as e:
			error("{} failed: {}".format(name, e))
		else:
			break
	else: # break not reached
		panic("No converters left")
		return -3
	dur = time.time() - st
	if size and dur:
		info("Processing took {:.1f} seconds at {:,.1f} MB/s".format(dur, size/10E6/dur))
	return 0
#


import socket
import datetime

HOST = '10.10.10.160'
PORT = 8888
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST,PORT))
s.send(input().encode())
print(s.recv(1024).decode())
s.close()


"""
모듈 사용하기
모듈
미리 만들어진 코드를 가져와 쓰는 방법
import 모듈이름
사용 방법: 모듈이름.모듈안의 구성요소

math.pi
random.choice()
"""

import math
r=10
sum=2*math.pi*r
print(sum)

import random
candidate=['가위','바위','보']
selected=random.choice(candidate)
for i in range(3):
    selected=random.choice(candidate)
    print(selected)

def getweb(url):
    """URL을 넣으면 페이지 내용을 들려주는 함수"""
    import urllib.request
    response=urllib.request.urlopen(url)
    data=response.read()
    decoded=data.decode('utf-8')
    return decoded

url=input('웹 페이지 주소? ')
content=getweb(url)
print(content)

"""
import math

수학과 관련된 기능
import random

무작위와 관련된 기능
import urllib.request

인터넷의 내용을 가져오는 기능
"""

"""
모듈 만들기
사용할 함수, 메소드 코드를 작성한 모듈 파일을 생성
모듈이 쓰일 파일에 import를 사용하여 모듈을 호출
사용 방법은 기존의 모듈과 동일
주의할 점은 사용자가 만든 모듈과 모듈을 쓸 파일이 같은 폴더에 있어야 한다.
"""


from matplotlib import pylab, pyplot as plt
import networkx as nx
import json
import os
import numpy as np
from adjacency import *
from scipy import signal

# SET THESE THREE PARAMS
'''
num:     [0-8] See "sets" variable.

         Set 2 has 3 agents, so you have to give options '20', '21', and '22'
         which correspond to the 1st, 2nd, and 3rd agents respectively.

         Set 7 has 2 agents, so you have to set'70' or '71' accordingly.

cen_num: 0:closeness (num == 0,20,21,22, 9), 1:degree (num == 1 and 6), 2: eigenvector (num == 3,4)

color:   Turn on region coloring
'''
num = 6
cen_num = 0
color = False

# ================================================================================================

agent_num = 0
if 20 <= num <= 30:
    agent_num = num % 20
    num = int(num / 10)

if 40 <= num <= 80:
    agent_num = num % 70
    num = int(num / 10)

sets = ['4',                        # 0
        '12_55_47',                 # 1
        '11_30_18',                 # 2
        '7_30_18_2',                # 3
        '10_30_18',                 # 4
        '13-2019-08-27-22-30-18',   # 5
        '16_30_18',                 # 6
        '9_47_10',                  # 7
        '5_55_47',                  # 8
        '10_55_47']                 # 9
frames = [[0, 60], [47, 80], [[55, 95], [70, 98],[70, 98]],
          [52, 98], [14, 50], [59, 85], [65, 72], [ [9, 35], [50, 98]],
          [58, 90], [10, 98]]
agent_IDs = [983, 2677, [2810, 2958, 2959], 1336, 3494, 1295, 1786, [[2562], [2564]], 1750, 868]
radius = [10, 20, 10, 20, 20, 10, 20, 10, 10, 10]
agent_labels = ['Black Car', 'White Car', 'White Car', 'White Bus',
                'White Truck', 'White Lorry', 'Motorbike', 'Scooter', 'Scooter', 'Motorbike']
centrality_labels = ['Closeness Centrality Value', 'Degree Centrality Value', 'Eigenvector Centrality Value']
thresholds = [[0, 35, 60], [47, 61, 80], [[55, 80, 95], [70, 80, 98], [70, 80, 98]], [52, 73, 98], [14, 40, 50],
              [59, 75, 85], [65, 68, 72], [ [9, 18, 35], [50, 75, 98]], [58, 80, 90], [24, 36]]

video_set = sets[num]
frame = frames[num] if (num != 2 and num != 7) else frames[num][agent_num]
agent_ID = agent_IDs[num]
rad = radius[num]
agent_label = agent_labels[num]
centrality_label = centrality_labels[cen_num]
x_lims = thresholds[num] if (num != 2 and num != 7) else thresholds[num][agent_num]

# filenames = os.listdir('data/')

# READ CSV FILE PANDAS
filepath = 'data/set' + str(video_set) + '_annotations_utm.json'

with open(filepath) as json_file:
    sample_annotation = json.load(json_file)

dataset_id = 1

to_list = []

for each_frame in sample_annotation:

    fr_id = each_frame['id']
    each_annotation = each_frame['annotations']
    for each_object in each_annotation:
        obj_id = each_object['classId']
        x = each_object['geometry']['position']['x']
        y = each_object['geometry']['position']['y']
        to_list.append([fr_id, obj_id, x, y, dataset_id])

to_array = np.asarray(to_list)
obj_IDs = np.unique(to_array[:, 1]).astype(int)
agent_idx = list(obj_IDs).index(agent_ID[agent_num]) if (num == 2 or num == 7) else list(obj_IDs).index(agent_ID)
degree_mat, adj_mats = generate_adjacency(to_array, rad)
weave_list = []
weave_list2 = []
weave_list3 = []
for fr, item in enumerate(adj_mats):
    G = nx.from_numpy_array(item['adj_matrix'])
    if cen_num == 0:
        g = nx.closeness_centrality(G)
    elif cen_num == 1:
        g = nx.degree_centrality(G)
    else:
        g = nx.eigenvector_centrality(G, max_iter=10000)
    # eg = nx.eigenvector_centrality(G)
    # print(fr,g[agent_idx])
    # if fr < 60:
    if frame[0] <= fr <= frame[1]:
        weave_list.append(g[0])  # num/id- 0,0, 1/1, 2/0, 3/0, 4/0
        weave_list2.append(g[agent_idx])
        # weave_list2.append(cg[7])
    # weave_list3.append(cg[8])

buffer2 = 7 if len(weave_list2) % 2 == 0 else 8
buffer = 7 if len(weave_list) % 2 == 0 else 8
if num != 6:
    weave_list2 = signal.savgol_filter(weave_list2, len(weave_list2)-buffer2, 3)
    # weave_list = signal.savgol_filter(weave_list, len(weave_list)-buffer, 3)

cmaps = ['#0038ff', '#00a4b2', '#4c6fb8', '#2c5167', '#9fd9ea']
LineThick = 5
FontSize = 40

fig, ax = plt.subplots(figsize=(11.0, 8.0))
x = np.arange(frame[0], frame[1] + 1)
y = weave_list2
# x = np.arange(frame[0], frame[1])
# y = weave_list2[2:] - 2 * weave_list2[:-1]
if len(agent_label.split()) > 1:
    agent_color = agent_label.split()[0]
else:
    agent_color = 'Gold'
ax.plot(x, y, linewidth=LineThick, label=agent_label, color='k')
if num == 0:
    ax.plot(x, weave_list, linewidth=LineThick, color='k')
    # ax.plot(range(frame[0], frame[1]+1), weave_list, linewidth=LineThick, label='Red Car', color='Tomato')
if color:
    for i in range(0, len(x_lims) - 1):
        ax.fill_between([x_lims[i], x_lims[i + 1]],
                        np.max(y[x_lims[i]-frame[0]:x_lims[i + 1]-frame[0]+1]) + 0.002,
                        np.min(y[x_lims[i]-frame[0]:x_lims[i + 1]-frame[0]+1]) - 0.002,
                        facecolor=cmaps[i], alpha=0.6, interpolate=True)
        # ax.fill_between([25, 35], np.max(y),np.min(y), facecolor='red', alpha=0.2, interpolate=True)
        # ax.fill_between([75, 85], np.max(y),np.min(y), facecolor='red', alpha=0.2, interpolate=True)
plt.grid(True)
plt.xlabel('Frame Number', fontsize=FontSize)
plt.ylabel(centrality_label.split()[0], fontsize=FontSize)
for tick in ax.xaxis.get_major_ticks():
    tick.label.set_fontsize(FontSize - 7)
for tick in ax.yaxis.get_major_ticks():
    tick.label.set_fontsize(FontSize - 7)
# Turn off tick labels
# ax.set_yticklabels([])
# legend = plt.legend(loc='lower left', bbox_to_anchor=(0, 1.01), ncol=2,
#                     borderaxespad=0, fontsize=FontSize - 5, fancybox=True,
#                     facecolor='green', framealpha=0.4)
# frame = legend.get_frame()
# frame.set_facecolor('green')
# frame.set_edgecolor('red')
# plt.savefig('images/' + video_set + '_' + agent_label + '.png', bbox_inches='tight')
# fig2, ax2 = plt.subplots(figsize=(15.0, 12.0))
# line, = ax2.plot(x, y,linewidth=LineThick, color='k')
# plt.grid(True)
# # plt.xlabel('Frame Number', fontsize=FontSize)
# # plt.ylabel(centrality_label.split()[0], fontsize=FontSize)
# for tick in ax2.xaxis.get_major_ticks():
#     tick.label.set_fontsize(FontSize - 7)
# for tick in ax2.yaxis.get_major_ticks():
#     tick.label.set_fontsize(FontSize - 7)
# y_init = list([0]*len(y))

# for n in range(len(x)):
#     line.set_data(x[:n], y[:n])
#     # ax2.axis([0, 100, 0, ])
#     fig.canvas.draw()
#     plt.savefig('video_materials/' + video_set + '_' + '{}.png'.format(n), bbox_inches='tight')

# plt.plot(range(frame[0], frame[1]+1), weave_list2, linewidth= LineThick, label=agent_label )
# if num==0:
#     plt.plot(range(frame[0], frame[1]+1), weave_list, linewidth= LineThick,label="Scooter" )
# plt.grid(True)
# plt.xlabel('Time (Frame Number)', fontsize=FontSize)
# plt.ylabel(centrality_label, fontsize=FontSize)
# plt.legend(loc='upper left', fontsize=FontSize)
plt.show()


#coding: utf8

from wedspider.spider import Spider
from others.sgf_spider import SgfSpider
from time import sleep

if __name__ == "__main__":

	'''
	for i in range(250):
		if(i > 220):
			queryurl = 'weiqi.sports.tom.com/hubo/sihuoying_'+str(i+1)+'.htm'
			s = SgfSpider(provider = 'tom_training')
			s.parser.assign('sgf_type', 1) # 1 - 死活题
			s.setUserAgent()
			s.append(url = queryurl)
			s.proceed()
			sleep(5)
	'''

	for i in range(40):
		if(i >= 0):
			if(i < 9):
				prefix = '0'
			else:
				prefix = ''

			queryurl = 'weiqi.sports.tom.com/shoujin/shoujin_0'+prefix+str(i+1)+'.htm'
			s = SgfSpider(provider = 'tom_training')
			s.parser.assign('sgf_type', 2) # 2 - 手筋
			s.setUserAgent()
			s.append(url = queryurl)
			s.proceed()
			sleep(5)

	'''
	queryurl = 'http://weiqi.sports.tom.com/php/listqipu.html'
	s = SgfSpider(provider = 'tom_game2', charset='gbk')
	s.parser.assign('sgf_type', 10) # 10 - 专业比赛
	s.parser.assign('sgf_level', 4) # 4 - 专业级别
	s.setUserAgent()
	s.append(url = queryurl)
	s.proceed(10000)
	'''

	'''
	queryurl = 'http://weiqi.sports.tom.com/php/listqipu2011.html';
	s = SgfSpider(provider = 'tom_updesc', charset='gbk')
	s.setUserAgent()
	s.append(url = queryurl)
	s.proceed(10000)
	'''


import pandas as pd
import numpy as np
import vk_api as vk
from bot import chatbot
from bot.chatbot import ChatBot
from contentGenerators import FriendFinder
import re
from sklearn import preprocessing

def get_year(date):
    if isinstance(date, str):
        if date.count('.') == 2:
            return re.split(r'\.', date)[2]
    return np.nan

def is_same_city(row, city_id):
    if isinstance(row['city'], dict):
        return row['city'].get('id') == city_id
    return np.nan

def get_lvl_by_id(user_id, dict_lvl_id):
    for key in dict_lvl_id:
        current_lvl_friend_ids = dict_lvl_id.get(key)
        if user_id in current_lvl_friend_ids:
            return key
    return np.nan

vk_session = vk.VkApi(token=...)
vk = vk_session.get_api()

basic_friends = vk.friends.search(user_id=..., fields="blacklisted,deactivated", count=10)
friens = basic_friends.get('items')
fr_id = []
for f in friens:
    a = f.get('deactivated')
    if not f.get('is_closed') and not f.get('blacklisted') and a != 'banned' and a != 'deleted':
        fr_id.append(f.get('id'))
lvlAndIds = {0: fr_id}

skok = 2
level = 0
for level in range(skok):
    mini_fr_id = []
    for friend in lvlAndIds.get(level):
        vals = vk.friends.search(user_id=friend, fields="blacklisted,deactivated", count=10).get('items')
        for f in vals:
            a = f.get('deactivated')
            if not f.get('is_closed') and not f.get('blacklisted') and a != 'banned' and a != 'deleted':
                mini_fr_id.append(f.get('id'))
    lvlAndIds[level + 1] = mini_fr_id
    level = level + 1

flatUserList = []
for key in lvlAndIds:
    flatUserList = flatUserList + lvlAndIds.get(key)

kek = pd.DataFrame(vk.users.get(user_ids=flatUserList, fields="sex,bdate,city,common_count"))

kek['lvl'] = kek.apply(lambda row: get_lvl_by_id(row['id'], lvlAndIds), axis=1)
kek['is_same_city'] = kek.apply(lambda row: is_same_city(row, 282), axis=1)
kek['is_same_city'] = kek['is_same_city'].fillna(kek['is_same_city'].astype(float).mean())

kek['year'] = kek['bdate'].apply(lambda each_date: get_year(each_date)).astype(int)
abs_dev = kek['year'].sub(2000).abs()
mean_abs_dev = abs_dev.mean()
cleared = kek['year'].sub(2000).abs().transform(lambda dev: mean_abs_dev if (dev > 50) else dev)
cleared_scaled = pd.Series(np.interp(cleared, (cleared.min(), cleared.max()), (0, 1)))
mean_cl_sc = cleared_scaled.mean()
weigts_age = cleared_scaled.fillna(mean_cl_sc).sub(1).abs()


# Information: https://clover.coex.tech/en/snippets.html#navigate_wait

import math
import rospy
from clover import srv
from std_srvs.srv import Trigger
import tf2_geometry_msgs
from tf.transformations import euler_from_quaternion

#rospy.init_node('flight')

get_telemetry = rospy.ServiceProxy('get_telemetry', srv.GetTelemetry)
navigate = rospy.ServiceProxy('navigate', srv.Navigate)
navigate_global = rospy.ServiceProxy('navigate_global', srv.NavigateGlobal)
set_position = rospy.ServiceProxy('set_position', srv.SetPosition)
set_velocity = rospy.ServiceProxy('set_velocity', srv.SetVelocity)
set_attitude = rospy.ServiceProxy('set_attitude', srv.SetAttitude)
set_rates = rospy.ServiceProxy('set_rates', srv.SetRates)
land = rospy.ServiceProxy('land', Trigger)

def navigate_wait(x=0, y=0, z=0, yaw=float('nan'), yaw_rate=0, speed=1, \
        frame_id='body', tolerance=0.2, auto_arm=False):

    res = navigate(x=x, y=y, z=z, yaw=yaw, yaw_rate=yaw_rate, speed=speed, \
        frame_id=frame_id, auto_arm=auto_arm)

    if not res.success:
        return res

print ("Take off 1 meter")
navigate_wait(z=2, frame_id='body', auto_arm=True)
# Wait for 3 seconds
print("Done")
rospy.sleep(3)

# Fly forward 1 m
#navigate_wait(x=1, frame_id='body')

# Land
#land()

def goal_republish(posedata):

    print("Received Simple")
    #do the transformation
    euler = euler_from_quaternion([posedata.pose.orientation.x, posedata.pose.orientation.y, posedata.pose.orientation.z, posedata.pose.orientation.w])
    print("Received Simple Goal: x=",posedata.pose.position.x," y=",posedata.pose.position.y," z=",posedata.pose.position.z," yaw =",euler[2]," frameID=",posedata.header.frame_id)
    navigate_wait( x=posedata.pose.position.x, y=posedata.pose.position.y, z=2, yaw=euler[2], frame_id=posedata.header.frame_id)
    print("Done")
    #rospy.sleep(3)

if __name__ == '__main__':

    rospy.init_node('republisher', anonymous=False)
    rospy.Subscriber("/move_base_simple/goal", tf2_geometry_msgs.PoseStamped, goal_republish)
    rospy.spin()


def sumatoria_cubica(n):
	res = 0
	for i in range(1,n+1):
		for j in range(1,i+1):
			for k in range(j,i+j+1):
				res+=1
	return res

def sumatoria_constante(n):
    res = 0
    res = (n*(n+1)*(n+2))/3
    return res


# -*- coding: utf-8 -*-

from zhihu_scrapy import db

zhihu = db.ZhihuMongo("zhihu")

def main():
    while zhihu["waitting_urls"].find_one():
        from scrapy.cmdline import execute
        execute("scrapy crawl zhihu_crawl".split())

if __name__ == "__main__":
    main()


from __future__ import unicode_literals

from ...zztlib import not_identical_strs, TestCase

class T(TestCase):
    def test0(self):
        s0, s1 = not_identical_strs()
        s0, s1 = (s0,), (s1,)
        self.assert_are_eq_not_same(s0, s1)

    def test1(self):
        s0, s1, s2 = not_identical_strs(r'abc', 3)
        s0, s1, s2 = (s0,), (s1,), (s2,)
        self.assert_are_eq_not_same(s0, s1, s2)


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 29 11:30:46 2017

@author: Deepak

Part A - Network A
"""

import sys
sys.path.append('../../')
sys.path.append('../')
import tensorflow as tf
from fdl_examples.datatools import input_data
import scipy
#import numpy as np
#import time, shutil, os
#import matplotlib.pyplot as plt
#from scipy import misc

# read in MNIST data --------------------------------------------------
mnist = input_data.read_data_sets("../../data/", one_hot=True)

# run network ----------------------------------------------------------

# Parameters
learning_rate = 0.01
training_epochs = 50 # NOTE: you'll want to eventually change this
batch_size = 100
display_step = 1

def inference(x,W,b):
    output = tf.nn.softmax(tf.matmul(x, W) + b)
    w_hist = tf.summary.histogram("weights", W)
    b_hist = tf.summary.histogram("biases", b)
    y_hist = tf.summary.histogram("output", output)
    return output

def loss(output, y):
    dot_product = y * tf.log(output)
    # Reduction along axis 0 collapses each column into a single
    # value, whereas reduction along axis 1 collapses each row
    # into a single value. In general, reduction along axis i
    # collapses the ith dimension of a tensor to size 1.
    xentropy = -tf.reduce_sum(dot_product, axis=1)
    loss = tf.reduce_mean(xentropy)
    return loss

def training(cost, global_step):
    tf.summary.scalar("cost", cost)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    train_op = optimizer.minimize(cost, global_step=global_step)
    return train_op

def evaluate(output, y):
    correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar("validation error", (1.0 - accuracy))
    return accuracy

if __name__ == '__main__':
#    if os.path.exists("logistic_logs/"):
#        shutil.rmtree("logistic_logs/")

    with tf.Graph().as_default():
        x = tf.placeholder("float", [None, 784]) # mnist data image of shape 28*28=784
        y = tf.placeholder("float", [None, 10]) # 0-9 digits recognition => 10 classes
        init = tf.constant_initializer(value=0)
        W = tf.get_variable("W", [784, 10],
                             initializer=init)
        b = tf.get_variable("b", [10],
                             initializer=init)
        output = inference(x,W,b)
        cost = loss(output, y)
        global_step = tf.Variable(0, name='global_step', trainable=False)
        train_op = training(cost, global_step)
        eval_op = evaluate(output, y)
        summary_op = tf.summary.merge_all()
        saver = tf.train.Saver()
        sess = tf.Session()

 #       summary_writer = tf.summary.FileWriter("logistic_logs/",
 #                                           graph_def=sess.graph_def)

        init_op = tf.global_variables_initializer()
        sess.run(init_op)

        # PLOTTING EACH DIGIT
        #x[7]=0, x[6]=1, x[13]=2, x[1]=3, x[2]=4, x[27]=5, x[26]=6, x[25]=7, x[9]=8, x[8]=9
        mini_x, mini_y = mnist.train.next_batch(30)

        num=mini_x[7]
        num0=num.reshape(28,28)

        num=mini_x[6]
        num1=num.reshape(28,28)

        num=mini_x[13]
        num2=num.reshape(28,28)

        num=mini_x[1]
        num3=num.reshape(28,28)

        num=mini_x[2]
        num4=num.reshape(28,28)

        num=mini_x[27]
        num5=num.reshape(28,28)

        num=mini_x[26]
        num6=num.reshape(28,28)

        num=mini_x[25]
        num7=num.reshape(28,28)

        num=mini_x[9]
        num8=num.reshape(28,28)

        num=mini_x[8]
        num9=num.reshape(28,28)

#       plt.imshow(num1)
#
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/0.jpg', num0)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/1.jpg', num1)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/2.jpg', num2)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/3.jpg', num3)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/4.jpg', num4)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/5.jpg', num5)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/6.jpg', num6)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/7.jpg', num7)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/8.jpg', num8)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/9.jpg', num9)

#        scipy.misc.imsave('outfile.jpg', image_array)

        # Training cycle
        for epoch in range(training_epochs):
            avg_cost = 0.
            total_batch = int(mnist.train.num_examples/batch_size)
            # Loop over all batches
            for i in range(batch_size):
                minibatch_x, minibatch_y = mnist.train.next_batch(batch_size)
            # Fit training using batch data
            sess.run(train_op, feed_dict={x: minibatch_x, y: minibatch_y})
            # Compute average loss
            avg_cost += sess.run(cost, feed_dict={x: minibatch_x, y: minibatch_y})/total_batch
            # Display logs per epoch step
            if epoch % display_step == 0:
                print("Epoch:", '%04d' % (epoch+1), "cost =", "{:.9f}".format(avg_cost))
                accuracy = sess.run(eval_op, feed_dict={x: mnist.validation.images, y: mnist.validation.labels})
                print("Validation Error:", (1 - accuracy))
                summary_str = sess.run(summary_op, feed_dict={x: minibatch_x, y: minibatch_y})
                #summary_writer.add_summary(summary_str, sess.run(global_step))
                #saver.save(sess, "logistic_logs/model-checkpoint", global_step=global_step)
        print("Optimization Finished!")
        accuracy = sess.run(eval_op, feed_dict={x: mnist.test.images, y: mnist.test.labels})

        #PLOTTING FIRST 10 WEIGHTS
        for wi in range (9):
            weights=sess.run(W)
            im=weights[:,wi]
            wim=im.reshape(28,28)
            filename = "C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/W_%d.jpg"%wi
            scipy.misc.imsave(filename, wim)
        print("Test Accuracy:", accuracy)

        with open('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/Accuracy.txt', 'w') as f:
            print('Accuracy = ',accuracy, file=f)
        f.close()


class SortingAlgorithms:

    def bubbleSort(self, arr):

        """
        Bubble Sort is an algorithm to sort a list of integers byt swapping adjacent elements
        if they are not in the correct order. Example: [3, 1] -> [1, 3].

        BigO Notation
        -------------
        N**2

        Parameters
        ----------
        arr: unsorted list of ints

        Returns
        -------
        arr: sorted list of ints
        """

        length = len(arr)
        if (length is 0):
            return arr

        for i in range(length - 1):
            for j in range(length - i - 1):

                if (arr[j] > arr[j + 1]):
                    arr[j], arr[j + 1] = arr[j + 1], arr[j]
        return arr

    def quickSort(self, arr):

        """
        Quick Sort is an alogrithm for sorting an integer list. This algorithm selects whatever number is at position 0.
        Once the pivot has been selected, every other number in the list is compared to the pivot. If the number
        is greater than, then it is put into a greater than list. If the number is less than, then it is put into
        a less than list. If there are multiples of the pivot, then the multiples are placed in the less than list.
        After the first round of sorting, recursion is used to sort the greater and lesser lists following the above
        steps until the entire list is sorted.

        BigO Notation
        -------------
        Depends on the pivot choice
        Average: N*logN
        Max: N**2

        Parameters
        ----------
        arr: unsorted list of ints

        Returns
        -------
        arr: sorted list of ints
        """

        length = len(arr)

        if(length is 0):
            return []

        pivotInt = arr[0]
        greaterThanPivot = []
        lessThanPivot = []

        for i in range(1, length):
            if(arr[i] > pivotInt):
                greaterThanPivot.append(arr[i])
            else:
                lessThanPivot.append(arr[i])

        return SortingAlgorithms.quickSort(self, lessThanPivot) + [pivotInt] + SortingAlgorithms.quickSort(self, greaterThanPivot)


import array as arr
# from array import * # To remove use of 'arr' everytime.

studentMarks = arr.array('i', [2,30,45,50,90])  # i represnts datatype of array which is int here.

# accessing array
print(studentMarks[3])

studentMarks.append(95)

# using for loop
for i in studentMarks:
    print(i)

# using while loop
i = 0
while i < len(studentMarks):
    print(studentMarks[i])
    i += 1


import os
from pathlib import Path
from collections import defaultdict
import torch
import torch.nn as nn
import torchaudio
from torch.utils.data import Dataset as TorchDataset
import numpy as np

from utils.audio import process_wav

# inherits Dataset from PyTorch
class Dataset(TorchDataset):

    def __init__(self, name, split, sample_rate, audio_length, tracks_list, num_segments, audio_proc_dir, mean, std):
        self.name = name
        self.split = split
        self.audio_proc_dir
        self.sample_rate = sample_rate
        self.audio_length = audio_length
        self.tracks_list = tracks_list
        self.num_segments = num_segments
        self.mean = mean
        self.std = std

        print(f"[{self.name} {self.split}]: Loaded {len(self.tracks_list)} audio segments")

    def indexer(self, ids, id2audio_path, id2gt, dataset, onepos=False):
        index = []
        tmp = []
        track_index = defaultdict(list)
        track_idx = 0
        clip_idx = 0
        for clip_id in ids:
            fp = id2audio_path[clip_id]
            label = id2gt[clip_id]

            if dataset == "magnatagatune":
                track_id = "".join(Path(fp).stem.split("-")[:-2])
                if track_id not in tmp:
                    tmp.append(track_id)
                    track_idx += 1
                fp = f"{track_idx}-{clip_id}-{self.sample_rate}.wav"
                fp = os.path.join(self.audio_proc_dir, self.split, fp)
            else:
                track_idx = clip_id
                clip_id = clip_idx
                fp = os.path.join(self.audio_proc_dir, fp)
                clip_idx += 1

            for s in range(self.num_segments):
                index.append([track_idx, clip_id, s, fp, label])
                track_index[track_idx].append([clip_id, s, fp, label])
        return index, track_index

    def loader(self, path):
        audio, sr = process_wav(self.sample_rate, path, False)
        return audio, sr

    def get_audio(self, fp):
        audio, sr = self.loader(fp)
        max_samples = audio.shape[0]
        # if sr != self.sample_rate:
        #     raise Exception("Sample rate is not consistent throughout the dataset")

        if max_samples - self.audio_length <= 0:
            raise Exception("Max samples exceeds number of samples in crop")

        # if np.isnan(audio).any():
        #     raise Exception("Audio contains NaN values")

        return audio

    def __len__(self):
        return len(self.tracks_list)

    def get_full_size_audio(self, fp):
        audio = self.get_audio(fp)

        # split into equally sized tensors of self.audio_length
        audio = torch.from_numpy(audio).reshape(1, -1)
        batch = torch.split(audio, self.audio_length, dim=1)

        # remove last, since it is not a full self.audio_length, and stack
        batch = torch.cat(batch[:-1])

        # reshape to B x 1 x N
        batch = batch.reshape(batch.shape[0], 1, -1)
        return batch

    def normalise_audio(self, audio):
        return (audio - self.mean) / self.std

    def denormalise_audio(self, norm_audio):
        return (norm_audio * self.std) + self.mean

    def sample_audio_by_track_id(self, track_id, batch_size=20):
        """
        Get audio samples based on the track_id (batch_size = num_samples)
        used for plotting the latent representations of different tracks
        """
        batch = torch.zeros(batch_size, 1, self.audio_length)
        for idx in range(batch_size):
            _, _, fp, _ = self.track_index[track_id][0]

            audio = self.get_audio(fp)

            start_idx = idx * self.audio_length

            # too large
            if (start_idx + self.audio_length) > audio.shape[0]:
                return None

            batch[idx, 0, :] = torch.from_numpy(audio[start_idx : start_idx + self.audio_length])
        return batch


import github_repo as gh
import json

OWNER = 'glass-bead-labs'
REPO = 'sensor-group'

sensor_group = gh.GitHubRepo(OWNER, REPO)

data = {}
dataset = []

for label in sensor_group.get_all_labels():
    issues = []
    for issue in sensor_group.get_issues_with_label(label):
        index = sensor_group.get_index(issue)
        size = sensor_group.get_num_comments(index) + 1
        issues.append({"name": issue, "size": size})
    dataset.append({"name": label, "children": issues})

issues = []
for issue in sensor_group.get_issues_without_label():
    index = sensor_group.get_index(issue)
    size = sensor_group.get_num_comments(index) + 1
    issues.append({"name": issue, "size": size})

dataset.append({"name": "no label", "children": issues})
data["name"] = OWNER + '/' + REPO
data["children"] = dataset

# Creates another JSON file.
with open ('bubble.json', 'w') as outfile:
    json.dump(data, outfile, indent=4)


from tqdm import tqdm
import pandas as pd
import numpy as np
import sys
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import random as rnd
from imblearn.over_sampling import SMOTE
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix, precision_recall_curve, auc, roc_auc_score, roc_curve, recall_score, classification_report
import sor_wc_wk_joint_noredundant as sor
import time
import cv_split
import pickle

scaler = StandardScaler()

pca = PCA(0.99)

pfile = 'test_data/test_' + sys.argv[1] + '.npz'

params = np.load(pfile)

print(params.files)

train_classes = params['train_classes']
test_classes = params['test_classes']
train_index = params['train_index']
test_index = params['test_index']

risk_class_files = ['Drought.csv', 'Earthquakes.csv', 'Explosions.csv', 'Floods.csv', 'Forest_and_Brush_Fire.csv', 'Hazardous_and_Toxic_Substance.csv', 'Landslides.csv', 'Lighting.csv', 'Snowstorms.csv', 'Tornado.csv', 'Tropical Storms.csv', 'Volcanoes.csv', 'Water_Pollution.csv']

risk_class_dict = {}
for i in range(len(risk_class_files)):
  risk_class_dict[i+1] = risk_class_files[i]

def remove_label(docs):
  for i in range(len(docs)):
    docs[i] = docs[i].replace('"1, ','').replace('"0, ','').replace("'0, ",'').replace("'0, ",'')
  return docs

risk_classes = {}
for risk_file in risk_class_files:
  risk_classes[risk_file] = pd.read_csv('../data/NYTimes_data/'+risk_file, header = None)[0].tolist()

non_risk_file = 'non_risk_docs.csv'

non_risk_class = pd.read_csv('../data/NYTimes_data/'+non_risk_file, header = None)[0].tolist()

X = []
Y = []

class_id = 1

for risk_file in risk_class_files:
  X += risk_classes[risk_file]
  Y += [class_id] * len(risk_classes[risk_file])
  class_id += 1

X += non_risk_class
Y += [0] * len(non_risk_class)

X = remove_label(X)

tfidf = TfidfVectorizer(ngram_range=(1,1), stop_words='english', token_pattern=u'(?ui)\\b\\w*[a-z]+\\w*\\b')

features = tfidf.fit_transform(X).toarray()
labels = Y

def run_test(features, labels, train_classes, test_classes, train_index, test_index):
    features = np.array(features)
    labels = np.array(labels)
    xtrain = features[np.isin(labels,train_classes),:]
    ytrain = labels[np.isin(labels,train_classes)]

    xtest = features[np.isin(labels,test_classes),:]
    ytest = labels[np.isin(labels,test_classes)]

    RR = 0
    R2Ri = 0
    RNR = 0
    NRNR = 0
    NRR = 0
    RiR = np.zeros(len(test_classes))
    RiNR = np.zeros(len(test_classes))

    X_train, X_test = xtrain[train_index], xtrain[test_index]
    y_train, y_test = ytrain[train_index], ytrain[test_index]

    scaler.fit(X_train)
    pca.fit(scaler.transform(X_train))
    X_train = pca.transform(scaler.transform(X_train))
    X_test = pca.transform(scaler.transform(X_test))
    xtest = pca.transform(scaler.transform(xtest))

    y_test_l = y_test.tolist()

    model = sor.HierarchicalClassifierModel(input_size = X_train[0].size, num_classes = len(risk_class_files), learning_rate = 1e-3, num_epochs = 1000, batch_size = 100, l1 = 0, l2 = 0, train_classes = train_classes)

    parameters = {'l1':np.logspace(-2,2,5), 'l2':np.append(np.logspace(-3,1,5),0)}
    splitter = cv_split.UnseenTestSplit()
    cmodel = GridSearchCV(model, parameters, cv = splitter, verbose=5, n_jobs = 10, error_score='raise')
    cmodel.fit(X_train, y_train.ravel())

    print('best params: l1=', cmodel.best_params_['l1'], 'l2=', cmodel.best_params_['l2'])

    model = sor.HierarchicalClassifierModel(input_size = X_train[0].size, num_classes = len(risk_class_files), learning_rate = 1e-3, num_epochs = 1000, batch_size = 100, l1 = cmodel.best_params_['l1'], l2 = cmodel.best_params_['l2'], train_classes = train_classes)

    model.fit(X_train, y_train)

    y_pred = model.predict(X_test, 0)
    y_pred_score = model.predict_score(X_test, 0)
    np.savetxt('test_results/' + sys.argv[1] + '_pca_joint_seen.out', y_pred_score)

    for j in range(len(y_test)):
            if y_test_l[j] >= 1:
                if y_pred[j] == 1:
                    RR += 1
                    y_pred_class = model.predict(X_test[j,:].reshape(1,-1), int(y_test[j]))
                    if y_pred_class == 1:
                      R2Ri += 1
                else:
                    RNR += 1
            else:
                if y_pred[j] == -1:
                    NRNR += 1
                else:
                    NRR += 1

    for classk in range(len(test_classes)):
            xtest_ri = xtest[ytest == test_classes[classk]]
            y_pred_ri = model.predict(xtest_ri, 0)
            y_pred_ri_score = model.predict_score(xtest_ri, 0)
            np.savetxt('test_results/' + sys.argv[1] + '_pca_joint_unseen_' + str(classk)+ '.out', y_pred_ri_score)

            for j in range(len(y_pred_ri)):
              if y_pred_ri[j] == 1:
                RiR[classk] += 1
              else:
                RiNR[classk] += 1

    print(RR, RNR, NRR, NRNR, RiR, RiNR, R2Ri)
    pickle.dump(model, open('test_results/trained_model_' + sys.argv[1] + '_pca_joint' + '.m', 'wb'))

print('Train classes:', train_classes)
print('Test classes:', test_classes)

run_test(features, labels, train_classes, test_classes, train_index, test_index)


 #playing around the list
l = [1,2,3,4,5]
l[2] = [1,4,6]
l2 = [100,100,100]
l[index] = l2[1]
index = round(len(l2))
l[index] = l2[1]
l[index:index] = 'a'
print(l*4)
l = l+l2
#l.append(l2)

print('spam' in l)
print('egg' not in l)

#with methods
l2 = l2.append([1,2,3])


# -*- coding: utf-8 -*-
"""
Created on Fri Apr  5 15:23:01 2013

Finite Difference Approximation and Convergence

@author: jspineda
"""
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from array_op import rotate #use np.roll

#interval [-2,6]

def f(xin):
    "Function f = x3 - 5x2 + x"
    x = np.array(xin)
    return np.power(x,3) - 5*np.power(x,2) + x

def df_sol(xin):
    "Analytic Derivative of f, see above"
    x = np.array(xin)
    return 3*np.power(x,2) - 10*x + 1

def df_for(x):
    "Forward derivative at x, using input descritization, ouput array has n-1"
    h = (rotate(x,-1) - x)[1]
    return ( f(rotate(x,-1)) - f(x))[:-1] / h  #does not include the latter edge point

def df_cent(x):
    "Central derivative at x, in step size h"
    h = (rotate(x,-1) - x)[1]
    return (f(rotate(x,-1)) - f(rotate(x,1)))[1:-1] / (2*h) #does not include edge points

def absicc(h,interval):
    "produce absicca on interval [-2,6] with given stepsize"
    n = int((interval[1] - interval[0])/ h)
    return h*np.arange(0,n+1) - 2

def quest(h):
    "Command for executing code for problem 1"
    interval = [-2,6.]
    x1 = absicc(h,interval) # define x axis on the interval with resolution h
    xf1 = x1[:-1]
    xc1 = x1[1:-1]
    x2 = absicc(h/2.,interval) # define x axis on the interval with resolution h/2
    xf2 = x2[:-1]
    xc2 = x2[1:-1]

    plt.clf()
    fig1 = plt.figure(1)
    plt.plot(xf1,df_for(x1) - df_sol(xf1),color='orange',label='Step Size: {0}'.format(h))
    plt.plot(xf2,df_for(x2) - df_sol(xf2),color='red',label='Step Size: {0}'.format(h/2.))
    plt.legend(loc='lower right')
    plt.title('Forward Differencing')
    pp1 = PdfPages('ws2_prob1_fig1.pdf')
    pp1.savefig(fig1)
    pp1.close()

    plt.clf()
    fig2 = plt.figure(2)
    plt.plot(xc1,df_cent(x1) - df_sol(xc1),color='darkgreen',label='Step Size: {0}'.format(h))
    plt.plot(xc2,df_cent(x2) - df_sol(xc2),color='blue',label='Step Size: {0}'.format(h/2))
    plt.legend(loc='center right')
    plt.title('Central Differencing')
    pp2 = PdfPages('ws2_prob1_fig2.pdf')
    pp2.savefig(fig2)
    pp2.close()

    return 'Done, see plots'


#!/usr/bin/env python
import datetime
import datetime
import numpy as np
import os
from PIL import Image, ImageFilter, ImageDraw, ImageFont
# import time
from bpassW import *
# from matplotlib import pyplot as plt
# from math import pi
from decimal import Decimal
from pkfndW2 import *
from skimage.draw import ellipse

# camera libraries
#from gpiozero import LED
#from picamera import PiCamera
#import io

def format_e(n):
	a = '%E' % n
	return a.split('E')[0].rstrip('0').rstrip('.') + 'E' + a.split('E')[1]

def kukicounter(imagefile):
	np.set_printoptions(precision=5,suppress=True)
	# ## take photo
	# led = LED(17)
	# led.on()
	# print ("LED on")
	# body = camera.get_frame()
	# now = datetime.datetime.now()
	# dstfilename = "Kyu" + now.strftime("%Y-%m-%d %Hh%Mm")
	# with open(dstfilename,'wb') as f:
	# 	f.write(body)
	# 	f.close()
	# print("LED off")
	# led.off()
	#imagefile = 'C:/Users/kuki/Downloads/071910am/1b01.jpeg'
	## pre-processing
	a = Image.open(imagefile) # open image
	ag = a.convert(mode="L") # mode=L (8bit, black and white) aka greyscale
	scf=0.5
	resize = tuple(int(i*scf) for i in ag.size)
	ag = ag.resize(resize, Image.ANTIALIAS)
	a = a.resize(resize, Image.ANTIALIAS)
	ag = np.asarray(ag)

	bg1 = np.mean(ag)
	# mean filter: subtract mean of all pixels from each pixel
	# noise-reduction
	imbr0 = ag-bg1
	imbr = np.abs(imbr0)
	#ag=Image.fromarray(ag)
	#median = ag.filter(ImageFilter.MedianFilter)
	#ag=np.asarray(ag)
	## cell dectection
	b0 = bpassW(imbr,lnoise=1,lobject=3) #put median for median filter

	var1=13
	ibg = np.mean(b0)+var1*np.std(b0)

	var2=20
	pk = pkfndW2(b0,ibg,var2)

	pk5=pk

	cc=np.zeros(len(pk5))
	cc[pk5.T[0]>0]=True
	cc[pk5.T[1]>0]=True
	pks=pk5

	no_cells = len(pks)*2

	print ('detected {:3.0f} object inbound out of total {:3.0f}'.format(np.sum(cc),len(cc)))

	# draw dot on top of cells
	a_draw = ImageDraw.Draw(a)
	for centroid in pks:
		y = centroid[0]
		x = centroid[1]
		e=[x-1,y-1,x+1,y+1]
		a_draw.ellipse(e,fill='rgb(100%,0%,0%)')
	del a_draw

	# area_px = 'imagesize'
	# px_size = 7.45 #um/px
	# area_mm = area_px*(px_size*0.001)**2
	# chamber_height = 0.1 #100nm
	# chamber_vol = (area_mm*chamber_height)/1000
	# density = format_e(Decimal(str(no_cells/chamber_vol)))
	# densitye = Decimal(str(no_cells/chamber_vol))
	# print ('cell density-- {:.2E}cells/ml'.format(densitye))

	# get a font
	#fnt = ImageFont.load_default()
	# get a drawing context
	#d = ImageDraw.Draw(a)
	# draw text
	#d.text((10,10), 'cell density-- {:.2E}cells/ml'.format(densitye), font=fnt, fill=(255,255,255))
	# draw text, full opacity
	#d.text((10,60), "World", font=fnt, fill=(255,255,255))
	file,ext=os.path.splitext(os.path.basename(imagefile))

	# create destination folder
	desktop = os.path.join(os.environ["USERPROFILE"], "Desktop")
	dstfolder = os.path.join('C:/Users/kuki/Downloads/071910am','count_results')
	if not os.path.exists(dstfolder): os.mkdir(dstfolder)
	# prevent overwriting counted image
	if not os.path.exists(file+'_counted.jpg'):
		a.save(os.path.join(dstfolder,file+'_counted.jpg'),format="JPEG")
	return no_cells

fol='C:/Users/kuki/Downloads/071910am'
ims = [os.path.join(fol,_) for _ in os.listdir(fol) if _.endswith('jpeg')]
for im in ims:
	no_cells = kukicounter(im)
print(no_cells)


from scrapy.spider import Spider
from scrapy.log import logger
from scrapy import Request
from ..items import VideoItem

'''
电影 电视剧 动漫 综艺
'''

class VideoSpider(Spider):
    name = "videoSpider"
    # 电影
    start_urls = ["http://m.77kp.com/vod-type-id-1-pg-1.html"]

    def parse(self, response):
        logger.info("---------------视频页当前数据的抓取  Begin------------------------------")
        title_divs = response.xpath("//div[@class='son_nav']")
        movice_divs = response.xpath("//div[@class='list mb']")

        for i, div in enumerate(title_divs):
            title = div.xpath("//li[@class='n1']/a/text()").extract()[i]
            logger.info(title)
            movices_lis = movice_divs[i].xpath(".//ul/li")

            for item_li in movices_lis:
                name = item_li.xpath(".//a/@title").extract()
                link = "http://m.77kp.com" + item_li.xpath(".//a/@href").extract_first()
                img = item_li.xpath(".//a/img/@src").extract()
                logger.info("视频名--->{}|视频详细链接--->{}|视频图片--->{}".format(name[0], link, img))
        logger.info("---------------视频页当前数据的抓取  End------------------------------")
        logger.info("---------------类型的抓取，同时构建类型页面的request---------------")
        # 所有类型的as
        type_as = response.xpath("//ul[@class='ui-tab fn-clear']/li/a")
        typeitem = {'bigtype': '', 'smailtype': '','name':name[0]}
        for a in type_as[:len(type_as) - 1]:

            type = a.xpath("text()").extract()
            type_link = "http://m.77kp.com" + a.xpath("@href").extract_first()
            if "片" in type[0]:
                typeitem['bigtype'] = '电影'
                typeitem['smailtype'] = type[0]
            else:
                typeitem['bigtype'] = '电视剧'
                typeitem['smailtype'] = type[0]
            yield Request(response.urljoin(type_link), callback=self.parse_video, meta={'typeitem': typeitem})
            logger.info("小的类型名--->{}|链接地址---->{}".format(type[0], type_link))
        logger.info("---------------类型的抓取  End------------------------------")
        # 测试
        # yield Request(response.urljoin("http://m.77kp.com/vod-type-id-5-pg-1.html"), callback=self.parse_video,
        #               meta={'typeitem': typeitem})

    def parse_video(self, response):
        links = []
        video_as = response.xpath("//div[@class='list mb bt']/ul/li/a")
        for a in video_as:
            name = a.xpath("@title").extract()
            link = "http://m.77kp.com" + a.xpath("@href").extract_first()
            img = a.xpath(".//img/@src").extract()
            links.append(link)
            logger.info("视频名--->{}|视频详细链接--->{}|视频图片--->{}".format(name[0], link, img[0]))

        next_page = "http://m.77kp.com" + response.xpath("//div[@class='ui-pages']/a/@href")[-2].extract()
        if next_page:
            logger.info("下一页网址----->{}".format(next_page))
            yield Request(response.urljoin(next_page), callback=self.parse_video)

        for link in links:
            yield Request(response.urljoin(link), callback=self.parse_videodetail, meta=response.meta)

    def parse_videodetail(self, response):
        typeitem = response.meta['typeitem']
        title = response.xpath("//div[@class='title']/text()").extract_first()
        performer = response.xpath("//div[@class='ui-detail fn-clear']/p/text()").extract_first()
        img = response.xpath("//div[@class='ui-img']/img/@src").extract_first()
        introduction = response.xpath("//div[@class='ui-detail-info']/text()").extract()[1]

        logger.info("大类型--->{}|小类型--->{}|视频名--->{}|演员--->{}|图片--->{}|简介--->{}".format(typeitem['bigtype'],
                                                                                typeitem['smailtype'], title,
                                                                                performer, img, introduction))
        video_urls = response.xpath("//div[@class='detail-list fn-clear']/a/@href").extract()

        online_urls = []
        thunder_urls = []
        magnet_urls = []

        for url in video_urls:
            if ".html" in url:
                online_urls.append("http://m.77kp.com" + url)
            elif "thunder://" in url:
                thunder_urls.append(url)
            elif "magnet:?xt" in url:
                magnet_urls.append(url)
        logger.info("在线播放地址--->{}|迅雷下载地址--->{}|磁力链接地址---->{}".format(online_urls, thunder_urls, magnet_urls))

        videoitem = VideoItem()
        videoitem['bigtype'] = typeitem['bigtype']
        videoitem['smailtype'] = typeitem['smailtype']
        videoitem['name'] = typeitem['name']
        videoitem['pic'] = img
        videoitem['performer'] = performer
        videoitem['introduction'] = introduction
        videoitem['videohtml']=response.url
        videoitem['videourls'] = online_urls
        videoitem['thunderurls'] = thunder_urls
        videoitem['magneturls'] = magnet_urls

        yield videoitem


import sys

import cv2 as cv

def inside(r, q):
    rx, ry, rw, rh = r
    qx, qy, qw, qh = q
    return rx > qx and ry > qy and rx + rw < qx + qw and ry + rh < qy + qh

def draw_detections(img, rects, thickness=1):
    for x, y, w, h in rects:
        # the HOG detector returns slightly larger rectangles than the real objects.
        # so we slightly shrink the rectangles to get a nicer output.
        pad_w, pad_h = int(0.15*w), int(0.05*h)
        cv.rectangle(img, (x+pad_w, y+pad_h), (x+w-pad_w, y+h-pad_h), (0, 255, 0), thickness)

def find_faces_haar(frame, face_cascade):
    gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(
        gray,
        scaleFactor=1.1,
        minNeighbors=5,
        minSize=(30, 30),
        flags=cv.CASCADE_SCALE_IMAGE
    )

    # Draw a rectangle around the faces
    for (x, y, w, h) in faces:
        cv.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    return frame

def find_faces_hog(img, hog):
    found, w = hog.detectMultiScale(img, winStride=(8,8), padding=(32,32), scale=1.05)
    found_filtered = []
    for ri, r in enumerate(found):
        for qi, q in enumerate(found):
            if ri != qi and inside(r, q):
                break
        else:
            found_filtered.append(r)
    draw_detections(img, found)
    draw_detections(img, found_filtered, 3)
    return img

if __name__ == '__main__':
    hog = cv.HOGDescriptor()
    hog.setSVMDetector(cv.HOGDescriptor_getDefaultPeopleDetector())

    face_cascade = cv.CascadeClassifier(sys.argv[1])

    cap = cv.VideoCapture(1)
    while True:
        ret, img = cap.read()

        # face detection using HOG
        # img = find_faces_hog(img, hog=hog)

        # HAAR
        img = find_faces_haar(img, face_cascade)

        # show the result.
        cv.imshow('capture', img)
        ch = cv.waitKey(1)
        if ch == 27:
            break
    cv.destroyAllWindows()


from numpy.lib.arraysetops import isin
from numpy.lib.function_base import average
from sklearn.metrics import mean_squared_error, mean_absolute_error, f1_score, r2_score
from sklearn.metrics import precision_score, recall_score
import numpy as np
import pandas as pd

def mae(app_gt,app_pred):
    return mean_absolute_error(app_gt,app_pred)

def rmse(app_gt, app_pred):
    return mean_squared_error(app_gt,app_pred)**(.5)

def f1score(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return f1_score(gt_temp, pred_temp)

def f1score_(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return f1_score(gt_temp, pred_temp, average='macro')

def recall(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return recall_score(gt_temp, pred_temp, )

def precision(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return precision_score(gt_temp, pred_temp, )

def relative_error(app_gt,app_pred):
    constant = 1
    numerator = np.abs(app_gt - app_pred)
    denominator = constant + app_pred
    return np.mean(numerator/denominator)

def r2score(app_gt,app_pred):
    # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score
    return r2_score(app_gt, app_pred)

def nde(app_gt,app_pred):
    # Normalized Disaggregation Error (NDE)
    # Inspired by http://proceedings.mlr.press/v22/zico12/zico12.pdf
    numerator = np.sum((app_gt-app_pred)**2)
    denominator = np.sum(app_gt**2)

    # if denominator < 1e-5:
    #     denominator = 1e-5

    frac = numerator/denominator
    if isinstance(frac, pd.Series):
        return np.sqrt(frac[0])
    else:
        return np.sqrt(frac)

def nep(app_gt,app_pred):
    # Normalized Error in Assigned Power (NEP)
    # Inspired by https://www.springer.com/gp/book/9783030307813
    numerator = np.sum(np.abs(app_gt-app_pred))
    denominator = np.sum(app_gt)

    # if denominator < 1e-5:
    #     denominator = 1e-5

    frac = numerator/denominator
    if isinstance(frac, pd.Series):
        return np.sqrt(frac[0])
    else:
        return np.sqrt(frac)


# -*- coding: utf-8 -*-
"""
Created on Sat Apr  1 10:06:23 2017

@author: samuel
"""
#加载所需模块，pandas提供excel支持，matplotlib.pyplot提供plt支持
import pandas as pd
import matplotlib.pyplot as plt
import os
#从excel中加载excel文件,目录自行修改
df = pd.read_excel(r'G:\Seafile\临时\Biodegradation of sulfur-rich oil\负离子excel\2.xlsx')
#将intensity转换为float类型
df['intensity']=df['intensity'].astype(float)
#按ppm筛选所需数据
df = df[(df.ppm>-2.0) & (df.ppm<2.0)]
#读取数据的所有化合物类，先剔除掉重复项，再将剩下的列举出来
y=df['class']
y=y.drop_duplicates()
y=y.reset_index()
m=len(y)
i=0
specie=0
#创建文件保存位置
script_dir = os.path.dirname(__file__)
results_dir = os.path.join(script_dir, '负2/')
if not os.path.isdir(results_dir):
    os.makedirs(results_dir)
#遍历上述操作找到的所有化合物类，分别绘制图谱
while i<m:
    specie=y.loc[i,'class']
    x=df[df['class']==specie]
    x['normalized']=x['intensity']/x['intensity'].sum()
    while x['normalized'].max()<100:
        x['normalized']=2*x['normalized']
    #分别绘图
    plt.figure(i)
    #设置图片格式
    font = {'family' : 'serif',
        'color'  : 'black',
        'weight' : 'normal',
        'size'   : 14,
        }
    plt.axis([0,60,0,16])
    plt.xlabel("Carbon Number",fontdict=font)
    plt.ylabel("DBE",fontdict=font)
    plt.text(1,14,s=specie,fontdict=font)
    plt.scatter(x['C'],x['DBE'],s=x['normalized'],alpha=0.8,edgecolors='white')
    sample_file_name = specie
    #保存图片
    plt.savefig(results_dir + sample_file_name,dpi=600)
    i=i+1


import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.datasets import mnist
import tensorflow as tf
gpus = tf.config.experimental.list_physical_devixe('GPU')
if gpus :
    try:
        tf.config.experimental.set_visible_devices(gpus[1],'GPU')
    except RuntimeError as e :
        print(e)

(x_train, y_train),(x_test, y_test) = mnist.load_data()

print(x_train.shape, y_train.shape)#(60000, 28, 28) (60000,)
print(x_test.shape, y_test.shape)#(10000, 28, 28) (10000,)

print(x_train[0])
print('y_train[0] : ', y_train[0])
print(x_train[0].shape)#(28, 28)

y_test1=y_test
x_train = x_train.reshape(60000,28,28,1)/255.
x_test = x_test.reshape(10000,28,28,1)/255.

#OnehotEncoding
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
encoder = OneHotEncoder()
y_train = encoder.fit_transform(y_train.reshape(-1,1)).toarray()
y_test = encoder.fit_transform(y_test.reshape(-1,1)).toarray()

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout
from tensorflow.keras.layers import BatchNormalization, Activation
from tensorflow.keras.regularizers import l1, l2, l1_l2
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(2,2), padding='same', strides=1, input_shape=(28,28,1)))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(32,(2,2), kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(Activation('relu')) # relu 계열 - he_normal  sigmoid,softmax계열 - xavier

model.add(Conv2D(32,(2,2), kernel_regularizer=l1(l1=0.01)))
model.add(Dropout(0.2))

model.add(Conv2D(32,(2,2), strides=2))
model.add(MaxPool2D(pool_size=2))

model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dense(10, activation='softmax'))

from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
es = EarlyStopping(monitor='val loss', patience=5)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
hist = model.fit(x_train, y_train, batch_size=128, epochs=5, validation_split=0.2, callbacks=[es])

loss, acc = model.evaluate(x_test, y_test, batch_size=200)
print('loss : ', loss)
print('accuracy : ', acc)

# loss :  0.09585337340831757
# accuracy :  0.9824000000953674


import byteUtils
import polyAlphabet

#By default, the alphabet is uppercase letters only
def encrypt(plain, key, alphabet=byteUtils.alphabetDefault, ignoreChars=byteUtils.ignoreCharsDefault):
	plainLength = len(plain)
	alphabetLength = len(alphabet)
	cipher = bytearray(plainLength)
	keyLength = len(key)
	j = 0
	for i in range(plainLength):
		if (plain[i] in ignoreChars):
			cipher[i] = plain[i]
			j = j + 1
		else:
			tableauRow = byteUtils.byteIndex(key[(i - j) % keyLength], alphabet)
			tableauColumn = byteUtils.byteIndex(plain[i], alphabet)
			cipher[i] = alphabet[(tableauRow + tableauColumn) % alphabetLength]
	return cipher

def decrypt(cipher, key, alphabet=byteUtils.alphabetDefault, ignoreChars=byteUtils.ignoreCharsDefault):
	cipherLength = len(cipher)
	alphabetLength = len(alphabet)
	plain = bytearray(cipherLength)
	keyLength = len(key)
	j = 0
	for i in range(cipherLength):
		if (cipher[i] in ignoreChars):
			plain[i] = cipher[i]
			j = j + 1
		else:
			tableauRow = byteUtils.byteIndex(cipher[i], alphabet)
			tableauColumn = byteUtils.byteIndex(key[(i - j) % keyLength], alphabet)
			plain[i] = alphabet[(tableauRow - tableauColumn) % alphabetLength]
	return plain

#Fully decrypts a Vigenere encryption (note: '69' is the ascii ['E'] character (the most common character in uppercase English texts with no spaces - use '101' ['e'] to decrypt an English text with no spaces).
def attack(cipher, mostLikelyChar=69, keyLength=-1, alphabet=byteUtils.alphabetDefault, ignoreChars=byteUtils.ignoreCharsDefault):
	#There appears to be no easy way to remove unwanted characters (spaces in particular)
	cipherCopy = bytearray(0)
	for b in cipher:
		if b in ignoreChars:
			continue
		else:
			cipherCopy.append(b)
	if keyLength == -1:
		keyLength = polyAlphabet.KasiskiExamination(cipherCopy)
	key = bytearray(keyLength)
	for i in range(keyLength):
		splitCipher = cipherCopy[i::keyLength]
		key[i] = alphabet[(byteUtils.byteIndex(byteUtils.byteFreqOrder(splitCipher)[0], alphabet) - byteUtils.byteIndex(mostLikelyChar, alphabet)) % len(alphabet) ]
	return decrypt(cipher, key)


import addressify
import unittest

from parameterized import parameterized

class AddressifyTest(unittest.TestCase):

    @parameterized.expand([
        ("none argument", None, AssertionError),
        ("empty argument", "", AssertionError),
        ("single word address", "MyAddress1", AssertionError),
        ("address without number", "Street street", Exception),
        ("address without street", "18 19", Exception),
        ("address with more than 5 components", "abyss street devoid of end no. 555", AssertionError)
    ])
    def test_invalid_address(self, name, input_data, expected_result):
        with self.assertRaises(expected_result):
            addressify.parse(input_data)

    @parameterized.expand([
        ("common german address", "Winterallee 3", {
            "street": "Winterallee",
            "housenumber": "3"
        }),
        ("german address with letter on number", "Blaufeldweg 123B", {
            "street": "Blaufeldweg",
            "housenumber": "123B"
        }),
        ("german adress with multiple words", "Am Bächle 23", {
            "street": "Am Bächle",
            "housenumber": "23"
        }),
        ("german address with multiple words and number with letter already spaced", "Auf der Vogelwiese 23 b", {
            "street": "Auf der Vogelwiese",
            "housenumber": "23 b"
        }),
        ("french address", "4, rue de la revolution", {
            "street": "rue de la revolution",
            "housenumber": "4"
        }),
        ("north american address", "200 Broadway Av", {
            "street": "Broadway Av",
            "housenumber": "200"
        }),
        ("spanish address", "Calle Aduana, 29", {
            "street": "Calle Aduana",
            "housenumber": "29"
        }),
        ("spanish address name has number on it", "Calle 39 No 1540", {
            "street": "Calle 39",
            "housenumber": "No 1540"
        }),
        ("made up address starting with number using letter", "574C, obliteration street", {
            "street": "obliteration street",
            "housenumber": "574C"
        }),
        ("chinese address using number abbreviation with period", "NO. 36 BAOSHAN JIUCUN", {
            "street": "BAOSHAN JIUCUN",
            "housenumber": "NO. 36"
        }),
        ("made up address with exactly 5 components", "forsaken abyss street no. 555", {
            "street": "forsaken abyss street",
            "housenumber": "no. 555"
        })
    ])
    def test_parse(self, name, input_data, expected_result):
        result = addressify.parse(input_data)
        self.assertDictEqual(expected_result, result)

    @parameterized.expand([
        ("north american addresss", "200 doomsday street", True),
        ("spanish address", "Calle del L'via L'Viaquez 3", False),
        ("german address", "Am Bächle 23", False),
        ("chinese address", " NO. 36 BAOSHAN JIUCUN", True)
    ])
    def test_guess_if_address_starts_with_number(self, name, input_data, expected_result):
        test_input = input_data.split()
        result = addressify.does_it_starts_with_number(test_input)
        self.assertEqual(expected_result, result)

    @parameterized.expand([
        ("abbreviation with period", "nO.", True),
        ("abbreviation without period", "no", True),
        ("no abbreviation", "on", False),
        ("unclear abbreviation", "-No.", False)
    ])
    def test_is_number_abbreviation(self, name, input_data, expected_result):
        result = addressify.is_number_prefix(input_data)
        self.assertEqual(expected_result, result)

    @parameterized.expand([
        ("only digits", "555", True),
        ("digits with letter", "1548B", True),
        ("digits with comma", "1548B,", True),
        ("no digits", "true", False)
    ])
    def test_internal_has_digit(self, name, input_data, expected_result):
        result = addressify.has_digit(input_data)
        self.assertEqual(expected_result, result)


occurences = {}

def countChars(line):
	for char in line:
		occurences[char] = occurences.get(char, 0) + 1

# Don't let the procedure call fool you, this is n^2 complexity, but meh it's challenge 2
with open('02_data.txt') as kruft:
	for line in kruft:
		countChars(line)

# 10 or less is rare, right?
print(''.join(i for i in occurences if occurences[i] < 10))


"""empty message

Revision ID: a0fbd4d0f7c3
Revises:
Create Date: 2020-11-20 15:39:08.444169

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'a0fbd4d0f7c3'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('player',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=80), nullable=False),
    sa.Column('club', sa.String(length=120), nullable=False),
    sa.Column('age', sa.Integer(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('player')
    # ### end Alembic commands ###


#!/usr/bin/env python3

import nltk
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold

import numpy as np
import pickle

######################################################################
CLASSIFIERS = ['maxent', 'naive_bayes', 'linear_svm', 'rbf01_svm', 'rbf02_svm']

def getROC(predictions, target_classes):
    tp, fp, tn, fn = [0] * 4
    for prediction,target_class in zip(predictions, target_classes):
        if (prediction == 1):
            if (target_class == 1):
                tp += 1
            else:
                fp += 1
        else:
            if (target_class < 1):
                tn += 1
            else:
                fn += 1

    tpr = tp/(tp + fn)
    fpr = fp/(fp + tn)
    acc = (tp + tn)/(tp + tn + fp + fn)
    return tpr, fpr, acc

def main():
    with open('nltk_featuresets.pickle', 'rb') as f:
        nltk_featuresets = pickle.load(f)
    with open('sklearn_featuresets.pickle', 'rb') as f:
        sklearn_featuresets = pickle.load(f)
    with open('target_classes.pickle', 'rb') as f:
        target_classes = pickle.load(f)

    trump_tweets_classifiers = []
    trump_tweets_classifiers.append((nltk.MaxentClassifier, "maxent"))
    trump_tweets_classifiers.append((nltk.NaiveBayesClassifier, "naive_bayes"))
    trump_tweets_classifiers.append((SVC(kernel='linear', probability=True), "linear_svm"))
    trump_tweets_classifiers.append((SVC(gamma=0.1), "rbf01_svm"))
    trump_tweets_classifiers.append((SVC(gamma=0.05), "rbf005_svm"))
    trump_tweets_classifiers.append((SVC(gamma=0.2), "rbf02_svm"))

    classifier_tpr = []
    classifier_fpr = []
    classifier_acc = []
    num_splits = 5
    cv = StratifiedKFold(n_splits=num_splits)
    for train, test in cv.split(nltk_featuresets, target_classes):
        training_set = [nltk_featuresets[i] for i in list(train)]
        testing_set = [nltk_featuresets[i] for i in list(test)]
        split_tpr = []
        split_fpr = []
        split_acc = []
        for classifier, name in trump_tweets_classifiers:
            if name == "maxent":
                trained_classifier = classifier.train(training_set, nltk.classify.MaxentClassifier.ALGORITHMS[0], max_iter=10)
            elif name == "naive_bayes":
                trained_classifier =  nltk.NaiveBayesClassifier.train(training_set)
            else:
                trained_classifier = classifier.fit(sklearn_featuresets[train], target_classes[train])

            if name == "maxent" or name == "naive_bayes":
                predictions = [trained_classifier.classify(nltk_featuresets[i][0]) for i in list(test)]
            else:
                predictions = trained_classifier.predict(sklearn_featuresets[test])

            tpr, fpr, acc = getROC(target_classes[test], predictions[:])
            split_tpr.append(tpr)
            split_fpr.append(fpr)
            split_acc.append(acc)
        classifier_tpr.append(split_tpr)
        classifier_fpr.append(split_fpr)
        classifier_acc.append(split_acc)

    for i,name in enumerate(CLASSIFIERS):
        print(name + " tpr: = {}".format(np.mean([row[i] for row in classifier_tpr])))
        print(name + " fpr: = {}".format(np.mean([row[i] for row in classifier_fpr])))
        print(name + " acc: = {}".format(np.mean([row[i] for row in classifier_acc])))

if __name__ == '__main__':
    main()


from __future__ import absolute_import

from ..vtask import VTask

from thrift.server.TNonblockingServer import TNonblockingServer
from thrift.transport.TSocket import TServerSocket

import time

class ThriftProcessorTask(VTask):
    LOOPLESS = True
    PROCESSOR = None

    def __init__(self, service):
        super(ThriftProcessorTask, self).__init__(service)
        assert self.PROCESSOR is not None
        self.processor = self.PROCESSOR(self.service)

class NBServerTask(VTask):
    DEFAULT_HOST = '0.0.0.0'
    DEFAULT_PORT = 0
    OPT_PREFIX = 'thrift'

    bound_host = bound_port = None

    def getProcessor(self):
        """Automatically find the ThriftProcessorTask subclass"""
        found = None
        for task in self.service.tasks:
            if isinstance(task, ThriftProcessorTask):
                assert found is None, "Multiple processor tasks! (%s, %s)" % \
                    (found.name, task.name)
                found = task
        assert found is not None, "No ThriftProcessorTask's found!"
        return found.processor

    def initTask(self):
        super(NBServerTask, self).initTask()

        self._stopped = False
        self.socket = TServerSocket(
            self.getTaskOption('host'), self.getTaskOption('port'))
        self.server = TNonblockingServer(
            self.getProcessor(), self.socket,
            threads=self.getTaskOption('threads'))
        self.server.prepare()
        self.bound_host, self.bound_port = \
            self.server.socket.handle.getsockname()
        self.logger.info("%s Server Started on %s:%s",
                         self.name, self.bound_host, self.bound_port)

    def stop(self):
        self.server.stop()
        self.server.close()
        self._stopped = True

    def _runloop(self):
        while not self.server._stop:
            self.server.serve()
        while not self._stopped:
            time.sleep(0.1)

    @classmethod
    def _addArguments(cls, ap):
        super(NBServerTask, cls)._addArguments(ap)
        ap.add_argument(cls._loptName('host'), default=cls.DEFAULT_HOST,
                        metavar='HOST',
                        help='Address to bind server to [%(default)s]')
        ap.add_argument(cls._loptName('port'), type=int, metavar='PORT',
                        default=cls.DEFAULT_PORT,
                        help='Port to run server on [%(default)s]')
        ap.add_argument(cls._loptName('threads'), type=int, default=10,
                        metavar='N',
                        help='Server Worker Threads [%(default)s]')


class py_solution:
    # def subset_helper(self,)

    def sub_sets(self, sset):
        if len(sset) == 0:
            return [[]]
        subset = self.sub_sets(sset[1:])
        return subset + [[sset[0]] + s for s in subset]

        # all_subsetts = []
        # for i in range(len(sset)):
        #     if i == 0:
        #         all_subsetts.append([])
        #     for j in range(len(sset)):


import os

PKG_NAME = 'vmemclient'
EXTRA_OPTIONS = {}

# Get the best setup()
try:
    from setuptools import setup
    EXTRA_OPTIONS = {
        'zip_safe': False,
    }
except ImportError:
    from distutils.core import setup

def get_contents(*args):
    contents = []
    for parts in args:
        filename = os.path.join(*parts)
        with open(filename, 'r') as fd:
            contents.append(fd.read().strip())
    return '\n\n'.join(contents)

# setup()
setup(
    name = PKG_NAME,
    version = get_contents(
        (PKG_NAME, 'data', 'version.dat'),
    ),
    description = 'Violin Memory Client Interface Library',
    long_description = get_contents(
        ('INTRO.rst',),
        ('PRODUCTS.rst',),
        ('README.rst',),
        ('CHANGELOG.rst',),
    ),
    author = 'Violin Memory, Inc.',
    author_email = 'opensource@vmem.com',
    url = 'http://www.vmem.com',
    packages = [x.format(PKG_NAME) for x in (
        '{0}', '{0}.core', '{0}.varray',
        '{0}.vshare', '{0}.concerto',
    )],
    classifiers = [
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: Apache Software License',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
    ],
    license = 'Apache Version 2.0',
    keywords = 'violin violinmemory vmem vmemclient rest',
    requires = [
    ],
    package_data = {
        PKG_NAME: [
            os.path.join('data', '*.dat'),
            os.path.join('data', '*.txt'),
        ],
    },
    **EXTRA_OPTIONS
)


#Plot do Notebook da Marina
def plot_histogram(data, plt, log=False):
    plt.hist(x=data, bins='auto', log=log)#, color='#0504aa', alpha=0.7, rwidth=0.85)

def plot_all_histograms(grouped_data, plt, attr, log=False):
    i = 1
    PLT_ROWS = 3
    PLT_COLUMNS = len(grouped_data)//PLT_ROWS+1

    plt.figure(figsize=(40, 100))

    sorted_grouped_data = sorted(
        grouped_data,  # iterates pairs of (key, corresponding subDataFrame)
        key=lambda x: len(x[1]),  # sort by number of rows (len of subDataFrame)
        reverse=True  # largest first
    )

    for stock_code, stock_code_data in sorted_grouped_data:
        # fit histogram into the grid
        plt.subplot(PLT_COLUMNS, PLT_ROWS, i)
        i += 1
        #

        plot_histogram(stock_code_data[attr], plt, log)
        plt.title(stock_code)