import asyncio
import random

import discord

#Token that connects with the bot
TOKEN = 'SECRET KEY'

client = discord.Client()

alive = True
channels = []

reply = ['Mijn gemeente heeft rond de 41.000 inwoners',
         'Je kan campussen van de UA vinden in de gemeente waar ik woon',
         'Ik woon niet ver van een Fort',
         'Als ge zat zijt, kunt ge op minder dan een half uur van de vizit tot bij mij thuis wandelen']

@client.event
async def on_message(message):
    if message.author == client.user:
        return
    print(message)
    command = message.content.split(' ')
    if command[0] == 'bot':
        channel = client.get_channel(int(command[1]))
        if command[2] == 'conv_biokot':
            await asyncio.sleep(12)
            await channel.send(
                'Awel das goe, dan babbel ik rechts van u wel met toffere mensen.')
        if command[2] == 'conv_hagar':
            await asyncio.sleep(16)
            await channel.send(
                'Ik hoor het al Thomas, gij vindt mij genen toffe. Ik zal me wel aan de rechterkant van de tafel '
                'zetten, zet gij u maar bij die middelste drie aan tafel. ‘T is al goe.')
        if command[2] == 'conv_all':
            await asyncio.sleep(2)
            await channel.send(
                'Awel das goe, dan babbel ik rechts van u wel met toffere mensen.')
            await channel.send(
                'Ik hoor het al Thomas, gij vindt mij genen toffe. Ik zal me wel aan de rechterkant van de tafel '
                'zetten, zet gij u maar bij die middelste drie aan tafel. ‘T is al goe.')
    print(message.channel.type)
    if not message.guild:
        if 'waar' in message.content or 'Waar' in message.content:
            await message.channel.send(reply[random.randint(0, len(reply)-1)])

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

if __name__ == '__main__':
    client.run(TOKEN)


"""
Toluwanimi Akindele
CMPUT 366 FALL 18
1440804

state s: gambler's capital {1,2,...,99}
action a: stakes {0,1,...,min(s, 100-s)}
ph: probability of the coin coming up heads
"""
import numpy as np
import matplotlib.pyplot as plt

def value_iteration(ph, goal_reward, states, theta, dph):
    value_func = np.zeros(states + 1)
    policy_func = np.zeros(states + 1)

    sweep = 0
    delta = 1.0
    while delta > theta:
        sweep += 1
        delta = 0.0

        # for each state s: gambler's capital {1,2,...,99}  -   s will range from 0 to 98
        for s in range(states):
            prev_value = value_func[s]
            max_val = 0.0
            pol = 0.0
            actions = list(range(min(s, 100-s) + 1))

            # each action leads to 2 possible states, win OR loss
            for a in actions:
                heads_state = s+a      # heads
                tails_state = s-a
                # if it reaches 100, reward = 1, otherwise 0
                if s+a >= 100:
                    heads_val = ph * (goal_reward + value_func[heads_state])
                else:
                    heads_val = ph * value_func[heads_state]
                tails_val = (1-ph) * value_func[tails_state]

                temp_val = heads_val + tails_val
                if temp_val >= max_val:
                    max_val = temp_val
                    pol = a

            value_func[s] = max_val
            policy_func[s] = pol

            # update delta to show how much error
            delta = max(delta, abs(prev_value - value_func[s]))

        if sweep in [1,2,3,32]:
            plt.plot(value_func[:99])
    plt.ylim(top=1)
    plt.ylim(bottom=0)
    plt.xlabel("Capital")
    plt.ylabel("Value Estimates")
    plt.suptitle("Value Function. Ph =" + str(dph))
    plt.show()
    plt.xlabel("Capital")
    plt.ylabel("Final Policy (Stake)")
    plt.plot(policy_func[:99])
    plt.suptitle("Final Policy. Ph =" + str(dph))
    plt.show()
    print("This is sweep:\t"+str(sweep))

def main():
    theta = 1e-5
    states = 100
    goal_reward = 1
    ph_list = [0.25, 0.55]

    for ph in ph_list:
        value_iteration(ph, goal_reward, states, theta,ph)

main()


#!/usr/bin/env python

# IMPORTS

import numpy as np
import matplotlib.pyplot as plt
import nibabel as nib
import sys
from scipy.ndimage.filters import gaussian_filter
from scipy.ndimage.filters import sobel
from itertools import product
from scipy.interpolate import RegularGridInterpolator
#from scipy.ndimage.filters import convolve
from scipy.signal import fftconvolve
from time import time
import os
from math import ceil
from copy import deepcopy
from pathos.pools import ProcessPool

# DESCRIPTION

usermanual = \
    """This utility tries to segment a 3D TOF (time-of-flight) MR image to
create a binary MR Angiogram (MRA)."""

# This module implements the algorithm originally described in:
# Jiaxin Wang, Shifeng Zhao, Zifeng Liu, Yun Tian, Fuqing Duan, and Yutong Pan,
# "An Active Contour Model Based on Adaptive Threshold for Extraction of
# Cerebral Vascular Structures", Computational and Mathematical Methods in
# Medicine, vol. 2016, Article ID 6472397, 9 pages, 2016.
# doi:10.1155/2016/6472397

# DEFINITIONS AND CODE

# Width of the regularised Dirac-delta function
EPSILON = 0.8
# Vesselness shape descriptor coefficients
V_ALPHA = None
V_BETA = None
V_GAMMA = None
# Intensity weighting for the vesselness function
TOFTRANS_MEAN = 90.0    # Mean of the TOF transition intensity range
TOFTRANS_WIDTH = 10.0   # Width of the TOF transition intensity range
# Vesselness lower threshold for Canny edge detector
V_THRESHOLD = 0.5
# Coefficient in [0.5, 1] for locally-specified dynamic threshold computation
K = 0.9
# Gaussian convolution kernel parameters
KERNEL_SIGMA = 0.5
KERNEL_RADIUS = int(ceil(3 * KERNEL_SIGMA))
# Energy function coefficients
ALPHA1 = 0.002
ALPHA2 = 0.004
BETA = 0.008
GAMMA = 0.04
MU_0 = 80
# Time increment
DT = 5
# Convergence threshold
PERCENT_CONVERGENCE = 1.0
# Regularisation constant
ETA = 1e-8
# Padding width along all axes to bypass boundary error
PAD_WIDTH = 5
# Maximum number of CPU cores allowed for use
MAX_CPU = 2

def _h(x, epsilon=EPSILON):
    """Quasi-smooth Heaviside function."""
    res = 0.5 + np.arctan(x / epsilon) / np.pi
    #res[x < -EPSILON] = 0.0
    #res[x > EPSILON] = 1.0
    return res
    #return 0.5 + np.arctan(x / epsilon) / np.pi

def _delta(x, epsilon=EPSILON):
    """Quasi-smooth Dirac delta function"""
    #return np.where(np.abs(x) > EPSILON, 0,
    #                epsilon / (epsilon ** 2 + x ** 2) / np.pi)
    return (epsilon / (epsilon ** 2 + x ** 2)) / np.pi

def _div(vfield):
    """Calculates the divergence of a vector field."""
    return np.sum(np.stack([np.gradient(vfield[..., i], axis=i)
                            for i in range(vfield.shape[-1])],
                           axis=vfield.ndim-1), axis=-1)

# Modified from source: https://stackoverflow.com/questions/31206443/
# numpy-second-derivative-of-a-ndimensional-array
def _hessian(x):
    """
    Calculate the hessian matrix with finite differences
    Parameters:
       - x : ndarray
    Returns:
       an array of shape x.shape + (x.ndim, x.ndim)
       where the array[... i, j] corresponds to the second derivative x_ij
    """
    x_grad = np.gradient(x)
    hessian = np.empty(x.shape + (x.ndim, x.ndim), dtype=x.dtype)
    for k, grad_k in enumerate(x_grad):
        # iterate over dimensions
        # apply gradient again to every component of the first derivative.
        tmp_grad = np.gradient(grad_k)
        for l, grad_kl in enumerate(tmp_grad):
            hessian[..., k, l] = grad_kl
    return hessian

def _laplacian(sfield):
    """Calculates the Laplacian of an n-dimensional scalar field."""
    return _div(_grad(sfield))

def _vesselness(Ra, Rb, S, eigvals, alpha=None, beta=None, gamma=None,
                img=None):
    """Calculates the vesselness score based on indicators of structuredness
    derived from the eigenanalysis of the local Hessians."""

    # These parameter settings looked intuitive to me, albeit they have not been
    # mentioned in the literature
    if alpha is None:
        alpha = np.std(Ra[np.nonzero(Ra)])
    if beta is None:
        beta = np.std(Rb[np.nonzero(Rb)])
    if gamma is None:
        gamma = np.std(S[np.nonzero(S)])

    res = np.zeros_like(Rb)
    roi = np.where(np.logical_and(eigvals[..., 1] <= 0,
                                  eigvals[..., 2] <= 0))
    if img is None:
        # Frangi's definition
        res[roi] = (1.0 - np.exp(-(Ra[roi] ** 2) / (2 * alpha ** 2))) * \
                   np.exp(-(Rb[roi] ** 2) / (2 * beta ** 2)) \
                   * (1.0 - np.exp(-(S[roi] ** 2) / (2 * gamma ** 2)))
    else:
        # With intensity weighting (only if the TOF is homogeneous)
        res[roi] = (1.0 - np.exp(-(Ra[roi] ** 2) / (2 * alpha ** 2))) * \
                   np.exp(-(Rb[roi] ** 2) / (2 * beta ** 2)) \
                   * (1.0 - np.exp(-(S[roi] ** 2) / (2 * gamma ** 2))) \
                   * (0.5 + 1.0/np.pi * np.arctan((img[roi] - TOFTRANS_MEAN) /
                                                  TOFTRANS_WIDTH))
    return res

def _shift(img, dirs, fill_value=0):
    """Shifts an N-D image with the specified extent along each dimension.
    Linear interpolation is used to translate the image. The output has the same
    size and shape as the input. Pixels outside the original image domain are
    filled with a constant value."""

    _dirs = np.asarray(dirs)
    assert img.ndim == _dirs.size, \
        "The inputs must have identical dimensionality."

    # Set up interpolator
    axes = tuple(np.arange(0, i) for i in img.shape)
    ipol = RegularGridInterpolator(axes, img, bounds_error=False,
                                   fill_value=fill_value, method='linear')

    # Calculate new coordinates
    new_axes = []
    for k in range(_dirs.size):
        new_axes.append(np.asarray(axes[k]) - dirs[k])

    # Return shifted image
    return ipol(np.stack(np.meshgrid(*tuple(new_axes), indexing='ij'))
                .reshape(_dirs.size, -1).T).reshape(img.shape)

# The implementation of the N-D Canny edge detector was based on the following
# description:
# http://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/
# py_imgproc/py_canny/py_canny.html

def cannyND(img, sigma=1, minval=None, maxval=None, eta=1e-8):
    """Canny edge detection for N dimensional images."""

    dim = img.ndim

    # Gaussian filtering
    _img = gaussian_filter(img, sigma=sigma)

    # Sobel filtering in all N directions
    sfimg = np.stack([sobel(_img, axis=i) for i in range(dim)], axis=dim)
    magnitudes = np.linalg.norm(sfimg, axis=-1)

    # Find local maxima of the gradient magnitude
    # pdirs: principal directions (neighbourhood in N dimensions)
    pdirs = np.stack(product(*((-1, 0, 1),) * dim))
    pdirs = pdirs[np.any(pdirs != 0, axis=-1)]
    pdirs = pdirs[:pdirs.shape[0]/2, :]
    nbix = np.argmax(np.abs(np.sum(sfimg[..., np.newaxis, :] * pdirs, axis=-1)
                     / np.linalg.norm(pdirs + eta, axis=-1)
                     / np.repeat(magnitudes[..., np.newaxis] + eta,
                                 pdirs.shape[0], axis=-1)), axis=dim)
    edges = np.zeros_like(magnitudes)
    for k, direction in enumerate(pdirs):
        current_voxels = magnitudes[np.where(nbix == k)]
        ref1 = _shift(magnitudes, direction)[np.where(nbix == k)]
        ref2 = _shift(magnitudes, -direction)[np.where(nbix == k)]
        edges[np.where(nbix == k)] = \
            np.logical_and(current_voxels > ref1, current_voxels > ref2)\
                .astype(np.int8)
        # Release memory
        del current_voxels
        del ref1
        del ref2
    magnitudes *= edges

    # Set default values of minval and maxval
    if minval is None:
        minval = np.percentile(magnitudes[np.nonzero(magnitudes)], 50)
        print ("Canny lower threshold value (minval): {0:0.03f}".format(minval))
    if maxval is None:
        maxval = np.percentile(magnitudes[np.nonzero(magnitudes)], 95)
        print ("Canny upper threshold value (maxval): {0:0.03f}".format(maxval))

    # Handle user error
    if maxval < minval:
        print ("WARNING: minval < maxval. Automatic correction: "
               "minval = maxval.")
        minval = maxval

    # Histeresis thresholding
    edges = np.where(magnitudes > minval, 1, 0)
    edges_certain = np.where(magnitudes > maxval, 1, 0)
    nb_exploration = np.zeros(edges.shape + (pdirs.shape[0],))
    for k, direction in enumerate(pdirs):
        nb_exploration[..., k] = _shift(edges_certain, direction)
    edges *= np.any(nb_exploration, axis=-1)

    return edges

def _kernel(img, sigma=KERNEL_SIGMA, radius=KERNEL_RADIUS):
    """Localised Gaussian convolution kernel."""

    dim = img.ndim
    if radius is not None:
        kernel = np.stack(np.meshgrid(
            *(np.linspace(-radius, radius, 2 * radius + 1),) * dim,
            indexing='ij'), axis=dim).astype(np.float64)
    else:
        kernel = np.stack(
            np.meshgrid(*tuple(np.linspace(-i/2, i/2, i) for i in img.shape),
                        indexing='ij'), axis=dim).astype(np.float64)
    kernel = np.linalg.norm(kernel, axis=-1)
    kernel = np.exp(-(kernel ** 2) / (2 * sigma ** 2)) \
             / (np.sqrt(2 * np.pi * sigma ** 2) ** dim)

    return kernel

def _g(x):
    """Regularised gradient map for geodesic active contour and nonlinear
    diffusion."""
    return np.divide(1.0, 1.0 + x ** 2)

def _grad(sfield):
    """Gradient of a scalar field"""
    return np.stack(np.gradient(sfield), axis=sfield.ndim)

def acm(tofimg, eta=1e-8):
    """
    :param ndarray tofimg: 3D bias-corrected TOF (time-of-flight) image.
    """

    # 1. Initialise vessel locations and their approximate boundaries
    # (Frangi's multi-scale vessel enhancement algorithm)

    R_multiscale = []
    scales = np.linspace(0, 1, 2, endpoint=True)
    for i, scale in enumerate(scales):

        # Update status
        print ("Scale {0:0.02f} px ({1:d}/{2:d}):"
               .format(scale, i+1, scales.size))

        # 1.1 Obtain the Hessian matrix for all voxels, perform eigenvalue
        # decomposition and order the eigenpairs by the magnitude of the
        # eigenvalues (the order is ascending)
        print ("Calculating Hessians...")
        smimg = gaussian_filter(tofimg, scale)
        eigvals, eigvects = np.linalg.eig(_hessian(smimg))
        eigval_order = np.argsort(np.abs(eigvals), axis=-1)
        grids = np.ogrid[[slice(0, i) for i in eigvals.shape]]
        eigvals = eigvals[tuple(grids)[:-1] + (eigval_order,)]
        grids = np.ogrid[[slice(0, i) for i in eigvects.shape]]
        eigvects = eigvects[tuple(grids)[:-1]
                            + (np.expand_dims(eigval_order,
                                              axis=smimg.ndim),)]

        # 1.2 Define shape descriptors
        Ra = np.abs(eigvals[..., 1].astype(np.float64)) \
             / np.abs(eigvals[..., 2].astype(np.float64) + eta)
        Ra[~np.isfinite(Ra)] = 0
        Rb = np.abs(eigvals[..., 0].astype(np.float64)) \
             / np.sqrt(np.abs(eigvals[..., 1].astype(np.float64))
                       * np.abs(eigvals[..., 2].astype(np.float64)) + eta)
        Rb[~np.isfinite(Rb)] = 0
        S = np.linalg.norm(eigvals.astype(np.float64), axis=-1)

        # 1.3 Calculate vesselness score
        print ("Calculating vesselness...")
        R_multiscale.append(
            _vesselness(Ra, Rb, S, eigvals, alpha=V_ALPHA, beta=V_BETA,
                        gamma=V_GAMMA, img=smimg))

    # Select maximum vesselness value from all scales
    R = np.max(np.stack(R_multiscale, axis=smimg.ndim), axis=smimg.ndim)

    # 1.4 Run Canny edge detection to initialise the contour
    #print ("Running N-dimensional Canny algorithm...")
    #R_th = np.copy(R)
    #R_th[R < V_THRESHOLD] = 0
    #contour = cannyND(R_th, sigma=0.5, eta=ETA)

    # 1.5 Initialise the level-set function using both vesselness and contour
    print ("Initialising level-set function...")
    phi = np.where(R < V_THRESHOLD, -3 * EPSILON, 3 * EPSILON)
    #phi[contour != 0] = 0

    # 2. Run active contour segmentation
    # 2.1 Calculate kernel function
    kernel = _kernel(tofimg, sigma=KERNEL_SIGMA, radius=KERNEL_RADIUS)

    # 2.2 Calculate edge function
    smimg = gaussian_filter(tofimg, sigma=3)
    edge = _g(np.linalg.norm(_grad(smimg), axis=-1))

    iteration = 0
    e_change = -1
    while e_change < 0:

        # Update status
        start_t = time()
        iteration += 1
        print ("Starting iteration No. {}...".format(iteration))

        # 2.2 Calculate locally-specified dynamic threshold
        phi_h = _h(phi, epsilon=EPSILON)
        placement = (-KERNEL_RADIUS,) * kernel.ndim
        mu = K * fftconvolve(phi_h * R, kernel, mode='same') \
             / (fftconvolve(phi_h, kernel, mode='same') + eta)

        # 2.3 Update phi to phi_k
        grad_phi = _grad(phi)
        delta_phi = _delta(phi, EPSILON)
        M1 = tofimg - MU_0
        M2 = R - mu
        M3 = _laplacian(phi) - \
             _div(np.divide(grad_phi,
                            np.repeat(
                                np.linalg.norm(grad_phi + ETA, axis=-1)
                                [..., np.newaxis],
                                axis=-1, repeats=grad_phi.shape[-1])))

        phi_k = phi + DT * (ALPHA1 * M1 * delta_phi + ALPHA2 * M2 * delta_phi
                            + GAMMA * M3)

        # 2.4 Update phi using phi_k
        grad_phi = _grad(phi_k)
        phi = phi_k + DT * delta_phi * BETA * _div(
            np.repeat(edge[..., np.newaxis],
                      repeats=grad_phi.shape[-1], axis=-1)
            * np.divide(grad_phi,
                        np.repeat(np.linalg.norm(ETA + grad_phi, axis=-1)
                                  [..., np.newaxis], axis=-1,
                                  repeats=grad_phi.shape[-1])))

        # 2.5 Calculate system total energy
        integral_1 = np.sum(M1 * phi_h)
        integral_2 = np.sum(M2 * phi_h)
        integral_3 = np.sum(edge * np.linalg.norm(_grad(phi_h), axis=-1))
        P = np.sum(0.5 * (np.linalg.norm(_grad(phi), axis=-1) - 1) ** 2)
        if iteration > 1:
            energy_old = energy
            energy = - ALPHA1 * integral_1 - ALPHA2 * integral_2 \
                     + BETA * integral_3 + GAMMA * P
            e_change = (energy - energy_old) / energy_old * 100.0
            print ("Total energy: {0:0.04f}, change: {1:0.03f} %. "
                   "Elapsed time: {2:0.01f} s.".format(energy, e_change,
                                                       time()-start_t))
            if np.abs(e_change) <= PERCENT_CONVERGENCE:
                break
        else:
            energy = -ALPHA1 * integral_1 - ALPHA2 * integral_2 \
                     + BETA * integral_3 + GAMMA * P
            print ("Total energy: {0:0.04f}. Elapsed time: {1:0.01f} s."
                   .format(energy, time()-start_t))
    return phi

def _filter_solitary(segmentation):
    """Removes solitary points from the segmentation. The input should be a
    boolean mask (with values 1 for vessel, 0 for non-vessel) derived from the
    level-set function."""

    dim = segmentation.ndim
    pdirs = np.stack(product(*((-1, 0, 1),) * dim))
    pdirs = pdirs[np.any(pdirs != 0, axis=-1)]
    pdirs = pdirs[:pdirs.shape[0] / 2, :]
    filter = np.where(segmentation > 0, 1, 0)
    nb_exploration = np.zeros(filter.shape + (pdirs.shape[0],))
    for k, direction in enumerate(pdirs):
        nb_exploration[..., k] = _shift(segmentation > 0, direction)
    filter *= np.any(nb_exploration, axis=-1).astype(np.int8)

    return filter

def parallel_job(imfile):
    """A sub-routine that is called by each parallel workers."""

    # Update status
    print ("Processing {}...".format(imfile))

    try:
        # Load image
        mri = nib.load(imfile)
        hdr = mri.header
        img = mri.get_data()
        # Pad image temporarily with emtpy slices
        print ("Addig temporary padding to the image...")
        img = np.pad(img, pad_width=PAD_WIDTH, mode='constant',
                     constant_values=0)
        # Run segmentation
        print ("Running the segmentation...")
        phi = acm(img, eta=ETA)
        # Remove padding
        phi = phi[tuple(slice(PAD_WIDTH, phi.shape[i]-PAD_WIDTH)
                        for i in range(phi.ndim))]
        # Filter out any individual points from the segmentation
        print ("Filtering segmentation...")
        segm = _filter_solitary(np.where(phi > 0, 1, 0))

        # Save output
        outputdir, outputname = os.path.split(os.path.abspath(imfile))
        if outputdir == "":
            outputdir = os.getcwd()
        try:
            fname = os.path.join(outputdir,
                                 outputname.replace(".nii.gz", "_phi.nii.gz"))
            nib.save(nib.Nifti1Image(phi, hdr.get_best_affine(), hdr), fname)
            print ("SAVED: {}".format(fname))
        except:
            print("ERROR while saving {}.".format(fname))

        try:
            fname = os.path.join(outputdir,
                                 outputname.replace(".nii.gz", "_segm.nii.gz"))
            nib.save(nib.Nifti1Image(segm, hdr.get_best_affine(), hdr), fname)
            print ("SAVED: {}".format(fname))
        except:
            print("ERROR while saving {}.".format(fname))
        return 0

    except:
        return 1

def main(args):
    """Main program code."""

    # Validate file paths in image list
    imfiles = []
    for imfile in args:
        try:
            _ = nib.load(imfile).header   # low-cost load operation
            imfiles.append(imfile)
        except:
            print ("SKIPPED: {} could not be opened.".format(imfile))
            continue

    # Process the images
    n_cpu = min([len(imfiles), MAX_CPU])
    if n_cpu > 1:
        parpool = ProcessPool(nodes=n_cpu)
        err = parpool.map(parallel_job, imfiles)
    else:
        err = parallel_job(imfiles[0])

    if err == 0:
        print ("All tasks were successfully completed.")
    else:
        print ("There were {} error(s).".format(err))

if __name__ == "__main__":
    if len(sys.argv) > 1:
        main(sys.argv[1:])
    else:
        print (usermanual)
        print ("\nPlease specify an image in the command-line arguments.")
        exit(0)


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/sanhehu/Documents/GitHub/troposphere_mate-project/troposphere_mate/examples/nested_stack/tier_1_1_iam_policy.py
# Compiled at: 2020-02-13 14:13:44
# Size of source mod 2**32: 1094 bytes
from troposphere_mate import Template, Parameter, Output, Export, iam, helper_fn_sub
template = Template()
param_env_name = Parameter('EnvironmentName',
  Type='String')
template.add_parameter(param_env_name)
iam_ec2_instance_policy = iam.ManagedPolicy('IamPolicy',
  template=template,
  ManagedPolicyName=(helper_fn_sub('{}-web-server', param_env_name)),
  PolicyDocument={'Version':'2012-10-17',
 'Statement':[
  {'Sid':'VisualEditor0',
   'Effect':'Allow',
   'Action':[
    's3:Get*',
    's3:List*',
    's3:Describe*'],
   'Resource':'*'}]})
output_iam_ec2_instance_policy_name = Output('IamInstancePolicyArn',
  Value=(iam_ec2_instance_policy.iam_managed_policy_arn),
  Export=(Export(helper_fn_sub('{}-iam-ec2-instance-policy-arn', param_env_name))),
  DependsOn=iam_ec2_instance_policy)
template.add_output(output_iam_ec2_instance_policy_name)


# f = float(input('请输入: '))
# c = (f - 32) / 1.8
# print('%.1f h = %.1f s'%(f,c))

# import math

# radius = float(input('r: '))
# perimeter = 2 * math.pi * radius
# area = math.pi * radius * radius
# print('l: %.2f' % perimeter)
# print('s: %.2f' % area)

year = int(input('year: '))
is_leap = (year % 4 == 0 and year % 100 != 0 or year % 400 == 0)
print(is_leap)


from Protodeep.utils.parse import parse_optimizer, parse_loss, parse_metrics
from Protodeep.layers.Layer import Layer
from Protodeep.layers.Input import Input

def add(self, layer):
    self.layers.append(layer)

def build_graph(self, inputs, outputs):

    def get_next_level(graph, current_level, next_order, depth, way):
        next_level = []
        for curr in current_level:
            graph[curr][f'used{way}'] = 1
            for next_l in graph[curr][way]:
                next_level.append(next_l)
                if way == 'nexts':
                    graph[next_l]['order'] = next_order
                    graph[next_l]['depth'] = depth
                next_order += 1
        return next_level, next_order

    def order_graph(graph, inputs, outputs):
        """ BFS Heat Map With overidding """
        current = [i.layer.name for i in inputs]
        next_order = 1
        depth = 0
        while len(current) > 0:
            depth += 1
            current, next_order = get_next_level(
                graph,
                current,
                next_order,
                depth,
                'nexts'
            )
        #  run otherway to remove unused layers
        current = [o.layer.name for o in outputs]
        depth = 0
        while len(current) > 0:
            depth += 1
            current, next_order = get_next_level(
                graph,
                current,
                next_order,
                depth,
                'prevs'
            )

    if isinstance(inputs, list) is False:
        inputs = [inputs]
    if isinstance(outputs, list) is False:
        outputs = [outputs]

    ldict = Layer.layer_dico
    for name, layer in ldict.items():
        # print(f'{name}: locked:{layer.locked}')
        if not layer.locked:
            for nm, l in ldict.items():
                if name not in self.graph.keys():
                    self.graph[name] = {
                        'prevs': [],
                        'nexts': [],
                        'order': 0,
                        'depth': 0,
                        'usednexts': 0,
                        'usedprevs': 0,
                    }
                if l.input_connectors is layer.output_connectors:
                    self.graph[name]['nexts'].append(nm)
                    layer.output_connectors.next_layers.append(l)
                    if len(self.graph[name]['nexts']) > 1:
                        self.linear = False
                if layer.input_connectors is l.output_connectors:
                    self.graph[name]['prevs'].append(nm)

    order_graph(self.graph, inputs, outputs)
    self.flatten_graph = [
        ldict[ly]
        for ly
        in sorted(self.graph, key=lambda l: self.graph[l]['order'])
        if self.graph[ly]['usednexts'] == self.graph[ly]['usedprevs'] == 1
    ]
    for layer in self.flatten_graph:
        layer.locked = True

    self.inputs = [i.layer for i in inputs]
    self.outputs = [o.layer for o in outputs]

def build(self):
    for layer in self.flatten_graph[::-1]:
        self.weights.extend(layer.get_trainable_weights())
        self.gradients.extend(layer.get_gradients())

def compile(self, features_shape, loss="BinaryCrossentropy",
            optimizer="Adam", metrics=None):
    if self.linked is False:
        input_layer = Input((features_shape))
        connectors = input_layer()
        self.flatten_graph = []
        prev_layer = None
        self.flatten_graph.append(input_layer)
        self.inputs = [input_layer]
        self.outputs = [self.layers[-1]]
        for layer in self.layers:
            connectors = layer(connectors)
            if prev_layer is not None:
                prev_layer.output_connectors.next_layers.append(layer)
            self.flatten_graph.append(layer)
            prev_layer = layer
    self.logs = {}
    self.loss = parse_loss(loss)
    self.optimizer = parse_optimizer(optimizer)
    self.metrics = parse_metrics(metrics)
    self.build()
    self.optimizer.compile(self)


import json
import logging

from django.contrib.auth.decorators import login_required
from django.http import (
    Http404,
    HttpResponse,
    HttpResponseForbidden
)
from papermerge.core.models import (
    BaseTreeNode,
    Page,
    Access
)
from papermerge.core.models.kvstore import (get_currency_formats,
                                            get_date_formats, get_kv_types,
                                            get_numeric_formats)

logger = logging.getLogger(__name__)

@login_required
def metadata(request, model, id):
    """
    model can be either node or page. Respectively
    id will be the 'id' of either node or page.
    E.g.
    POST /metadata/page/55 # will update metadata for page id=55
    POST /metadata/node/40 # will update metadata for node id=40
    """
    if model == 'node':
        _Klass = BaseTreeNode
    else:
        _Klass = Page
    try:
        item = _Klass.objects.get(id=id)
    except _Klass.DoesNotExist:
        raise Http404("Node does not exists")

    kvstore = []

    if request.method == 'GET':
        for kv in item.kv.all():
            kvstore.append(kv.to_dict())
    else:
        if isinstance(item, BaseTreeNode):
            node = item
        else:
            node = item.document

        if request.user.has_perm(Access.PERM_WRITE, node):
            kv_data = json.loads(request.body)
            if 'kvstore' in kv_data:
                if isinstance(kv_data['kvstore'], list):
                    item.kv.update(
                        _sanitize_kvstore_list(kv_data['kvstore'])
                    )
        else:
            return HttpResponseForbidden()

    return HttpResponse(
        json.dumps(
            {
                'kvstore': kvstore,
                'currency_formats': get_currency_formats(),
                'date_formats': get_date_formats(),
                'numeric_formats': get_numeric_formats(),
                'kv_types': get_kv_types()

            }
        ),
        content_type="application/json"
    )

def _sanitize_kvstore_list(kvstore_list):
    """
    Creates a new dictionay only with allowed keys.
    """
    new_kvstore_list = []
    allowed_keys = [
        'id',
        'key',
        'value',
        'kv_type',
        'kv_format',
        'kv_inherited',
    ]

    for item in kvstore_list:
        sanitized_kvstore_item = {
            allowed_key: item.get(allowed_key, None)
            for allowed_key in allowed_keys
        }
        new_kvstore_list.append(sanitized_kvstore_item)

    return new_kvstore_list


#! Python3

import random
import logging
from train import train_and_accuracy

logging.debug('In network file.')

class Network():

    logging.debug('In network calss.')

    def __init__(self, para_choice = None):
        self.accuracy = 0
        self.mac_op = 0
        self.para_choice = para_choice
        self.network = {}
        self.kernel = []
        self.layers = []
        self.mac = 0
        self.mem = 0

    def create_random(self):
        "Create a random network."
        for key in self.para_choice:
            self.network[key] = random.choice(self.para_choice[key])

    def create_set(self, network):
        '''
        Set network properties.

        Args:
            network_no (int): in each generation there are fixed number of network which are created.

        '''

        logging.debug('In create set in network.')
        self.network = network

    def train(self,generation_no: int, network_no: int, time: str) -> None:
        '''Train the network and record the accuracy.

        Args:
            generation_no (int): number of generation (iteration for evolutionary algorithm).
            network_no (int): in each generation there are fixed number of network which are created.
            time: current time ("current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")")
         '''
        logging.debug('In Network, train function.')

        if self.accuracy == 0 and self.mac_op == 0 and self.kernel == [] and self.layers == [] and self.mac == 0 and self.mem == 0:
            self.accuracy, self.mac_op, self.kernel, self.layers, self.mac, self.mem = train_and_accuracy(self.network, generation_no, network_no, time)

    def get_network(self):
        return self.network

    def print_network(self):
        "Print a network."
        logging.info('This is with Max_pool as first two layers.')
        logging.info(f'Network Info: {self.network}, Accuracy: {self.accuracy}, Mac and Memory: {self.mac_op}, Kernel_list: {self.kernel}, Layers: {self.layers}, Mac: {self.mac}, Mem: {self.mem}')
        #logging.info(f"Network fitness: {(self.accuracy + self.mac_op)*100:.2f}")


# coding: utf-8

# In[1]:

import pandas as pd
iris_filename = "IRIS.csv"
iris = pd.read_csv(iris_filename, header=None, names=['sepal_length', 'sepal_width', 'petal_length', 'petal_width','target'])
iris.head()

# In[6]:

iris.describe()

# In[8]:

boxes = iris.boxplot(return_type='axes')
boxes

# In[9]:

#values at 10 percent and the 90 percent of the distribution of values
iris.quantile([0.1, 0.9])

# In[10]:

iris.target.unique()

# In[13]:

#count number of times the petal_length is more than the average
pd.crosstab(iris['petal_length'] > 1.21300, iris['petal_width'] > 0.464048)

# In[14]:

scatterplot = iris.plot(kind='scatter', x='petal_width', y='petal_length', s=64, c='blue', edgecolors='white')

# In[15]:

distr = iris.petal_width.plot(kind='hist', alpha=0.5, bins=20)

Python Data Science Essentials - Second Edition /data pipeline/exploratory_data_analysis.py


import requests
import re

tolurl = 'https://www.ximalaya.com/revision/album/v1/getTracksList?albumId=18943952&pageNum=3'
myHeader = {
    'User-Agent':
	'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0'
}
content = requests.get(tolurl,headers = myHeader)
jsonContent = content.json()
myDatab = jsonContent['data']['tracks']
#print(myDatab)
urlList = []
for i in myDatab:
    urlList.append('https://www.ximalaya.com'+i['url'])

print(urlList)


# -*- coding: utf-8 -*-
# ganben: this class defines the scheduler

import binascii
import os
import itertools
import copy
import random

# use serializable
import rlp

# use local
import classalloc

# auxillary funcs

# serializable data structs

class SchecStud(rlp.Serializable):
    #
    fields = [
        ('id', rlp.sedes.big_endian_int),
        # ('cmplan', rlp.sedes.big_endian_int),
        ('cmclass', rlp.sedes.big_endian_int),
        ('subjcomb', rlp.sedes.big_endian_int),
    ]

class ClassStud(rlp.Serializable):
    # in class with single sub stud
    fields = [
        ('id', rlp.sedes.big_endian_int),
        ('cmclass', rlp.sedes.big_endian_int),
        ('sub', rlp.sedes.big_endian_int),
    ]

class Teachers(rlp.Serializable):
    #
    fields = [
        ('tid', rlp.sedes.big_endian_int),
        ('subj', rlp.sedes.big_endian_int),
        ('cmclass', rlp.sedes.big_endian_int),
    ]
# mapping class between sub and subcomb
#
class MapCombSub:
    # use itertools to generate dict and inversed dict data structre

    def __init__(self, n_subs, m_picked):
        # n: a list of subs to pick
        # m: number of choices;
        #assert(m_picked>len(n_subs))
        #
        self.subs = n_subs
        self.comb_list = list(itertools.combinations(n_subs, m_picked))

    def if_contains(self, combs, sub):
        if self.comb_list[combs]:
            for el in self.comb_list[combs]:
                if el == sub :#they are all int(id, pk of original model)
                    return True

            return False

    def get_subs(self, comb):
        # return a tuples
        # (sub1, sub2, sub3)
        return self.comb_list[comb]

    def get_combs(self, sub):
        # input: subs: 1 - 3 or more
        # unordered list of sub in integer
        # return a list of combs in integer
        # if len(subs)>1:# fail to find out a way, skip
            # r = []
            # for e in subs:
                # r.append()
        # else:
        # l = copy.deepcopy(self.comb_list) #useless
        for it in self.comb_list:
            res = []
            for el in it:
                if el == sub:
                    res.append(it)

        return res

    @property
    def values(self):
        return range(len(self.comb_list))

class Croom(rlp.Serializable):
    fields = [
        ('rid', rlp.sedes.big_endian_int),
        ('cap', rlp.sedes.big_endian_int),
    ]

class TeachingSubs(rlp.Serializable):
    fields = [
        ('sid', rlp.sedes.big_endian_int),
        # ('nums', rlp.sedes.big_endian_int),
        ('is_bounded', rlp.sedes.boolean),
    ]

class SchecSetups(rlp.Serializable):
    #
    fields = [
        ('sum_weekly', rlp.sedes.big_endian_int),
        ('m_perday', rlp.sedes.big_endian_int),
        ('mx_rooms', rlp.sedes.big_endian_int),
        ('fix_subs', rlp.sedes.List(rlp.sedes.big_endian_int)),
        ('fix_weekly', rlp.sedes.big_endian_int), # simplified
        ('dyn_weekly', rlp.sedes.big_endian_int), # simplified
        ('min_members', rlp.sedes.big_endian_int),
        ('max_members', rlp.sedes.big_endian_int),
    ]

class BeforeSched(rlp.Serializable):
    #
    fields = [
        ('subs', rlp.sedes.CountableList(TeachingSubs)),
        ('teachers', rlp.sedes.CountableList(Teachers)),
        ('m_pick', rlp.sedes.big_endian_int),
        ('fix_subs', rlp.sedes.CountableList(TeachingSubs)),
        ('n_subs', rlp.sedes.big_endian_int),
        ('cmclasses', rlp.sedes.CountableList(classalloc.Cmclass)),
    ]

class UnitMember(rlp.Serializable):
    # mini unit
    fields = [
        ('stud', rlp.sedes.big_endian_int),
        # ('room', rlp.sedes.big_endian_int),
        ('sub', rlp.sedes.big_endian_int),
        # ('teacher', rlp.sedes.big_endian_int),
    ]

class SchedClass(rlp.Serializable):
    # sorted class unit
    fields = [
        ('sub', rlp.sedes.big_endian_int),
        ('room', rlp.sedes.big_endian_int),
        ('teacher', rlp.sedes.big_endian_int),
        ('studs', rlp.sedes.CountableList(UnitMember)),
    ]

class UnitClass():
    # methods to determine a complete or in-complete
    # mutable, to a immutable serializable sub class
    def __init__(self, pos = (0,0), room = 0):
        # pos1 pos2 is position of table matrix
        self.room = room
        # self.valid = False
        self.pos = pos
        self.cmcls = False # default is not a fix sub class
        self.studs = []
        self.sub = None
        self.teacher = None
        self.unused = []    # a swap stack of studs
        self.settled = 0
    # output a fixed result

    def setup(self, m_max, m_min, is_bound):
        # m_max = class member
        # m_min class member
        # is bound = selectable sub or forceable
        self.m_max = m_max
        self.m_min = m_min
        self.is_bound = is_bound

    @property
    def binary(self):
        if self.valid:
            uc = SchedClass(
                sub = self.sub,
                room = self.room,
                teacher = self.teacher,
                studs = self.studs
            )
            return rlp.encode(uc)
        else:
            raise Exception

    @property
    def valid(self):
        if not self.sub or not self.teacher:
            return False
        elif self.settled > self.m_min and self.settled == len(self.studs):
            return True
        else:
            return False

    @property
    def unused(self):
        return self.unused

    def add_stud(self, stud):
        self.unused.append(stud)

    def fill_fixed_stud(self, class_stud):
        # class_stud: the cmclass together stud list, without sub input
        if self.cmcls:
            #only work for fix cmcls class
            for s in class_stud: #copy these stud to class
                ss = {
                    'id':s.id,
                    'cmclass':self.cmcls,
                    'sub': self.sub
                }
                self.studs.append(ss)
            #TODO: maybe will fill with sub?

    def sched_stud(self, total_stud):
        #check if status ok
        # total_stud: un scheduded stud*sub list
        # use: the main thread: unitclass.sched_stud(current stud)
        if len(total_stud) > 0:
            for i in range(len(total_stud)):
                head = total_stud[i]
                if head.sub == self.sub and len(self.studs) < self.m_max:
                    self.studs.append(head)
                    self.settled += 1
                    total_stud.pop(i)

    def assign_teacher(self, teacher_id):
        self.teacher = teacher_id

    def assign_room(self, room_id):
        self.room = room_id

    def assign_fix_cmcls(self, cmcls):
        self.cmcls = cmcls

    def assign_sub(self, sub_id):
        self.sub = sub_id

class ClassTable:
    # this is a multi dimension array
    # can be use to swap between
    def __init__(self, setup):
        self.setup = setup # {.sum_weekly, .m_perday, mx_rooms, fix_sub list, fix_weekly int, dyn_weekly}
        # generate empty slots
        self.slots = [[] for i in range(5)]
        self.units = []
    @classmethod
    def gen_slots(n):
        # 5 day per week, fixed
        lists = [[] for i in range(5)]
        return lists

    def gen_fix_unitclass(self, subs, cmclass_list):
        # xx it with unit classes with position x,y, and room id for conflict resolving
        # generate the forceable first
        for i in subs:#range(self.setup['fix_sub']):
            for j in cmclass_list:
                u = UnitClass()
                u.sub = i.sid
                u.assign_fix_cmcls(j.cmid)
                u.studs = j.studs[:]
                li = [u] * self.setup['fix_weekly']
                self.units += li
        # not position/room conflict resolved

    def gen_dyn_unitclass(self, subs):
        # generate free picked classes
        # mapCS = MapCombSubs objects
        for i in subs:
            # should times n for each subs ?
            u  = UnitClass()
            u.sub = i.sid
            li = [u] * self.setup['dyn_weekly']
            self.units += li

    def slot_fill_fix(self):
        # each slot, fill with a list or enum kv store:?
        pass

    def class_fill(self, total_studs):
        for it in self.slots:
            for item in it:
                item.sched_stud(total_studs)

def gen_default_subs(n_fix=3, n_dyn=6):
    # generate default subs list1 fixed subs
    # subs list2: dyn subs list
    fix_list = []
    for i in range(n_fix):
        t = TeachingSubs(
            sid = i,
            is_bounded = True
        )
        fix_list.append(t)

    dyn_list = []
    k = random.randrange(10, 20)
    for i in range(n_dyn):
        t = TeachingSubs(
            sid = i + k,
            is_bounded = False
        )
        dyn_list.append(t)
    return fix_list, dyn_list

DEFAULT_SETUP = SchecSetups(
    sum_weekly = 5*8,
    m_perday = 8,
    mx_rooms = 10,
    fix_subs = 3,
    fix_weekly = 5,
    dyn_weekly = 5,
    min_members = 30,
    max_members = 45
)
# main class for perform multi level iterator
class Scheduler:
    # in mem , serializable input/output class
    def __init__(self, fn):
        # deserialize BeforeSched
        with open(fn, 'rb') as f:
            b = f.read()
        self.substuds = []
        self.before = rlp.decode(b, BeforeSched)
        self.setup = DEFAULT_SETUP
        # generate mapcombs
        self.mapsub = MapCombSub(self.before.subs, self.before.m_picked)
        # generate stud with subs from mapcombs
        for ic in self.before.cmclasses:
            # loop each class
            # cid = ic.id
            for it in ic:
                # iter three tuples
                sublist = list(self.mapsub.get_subs(it.subjcomb))
                st = []
                for sub in sublist:
                    cs = ClassStud(
                        id = it.id,
                        cmclass = ic.cmid,
                        sub = sub
                    )
                    st.append(cs)
                self.substuds += st * self.setup.dyn_weekly
        self.table = ClassTable(self.setup)
        # self.fix_list, self.dyn_list = gen_default_subs()
        self.fix_list = self.before.fix_subs
        self.dyn_list = self.before.subs

    def start_schec(self):
        #

        for item in self.before.cmclasses:

            self.table.gen_fix_unitclass(self.fix_list, item)   # the sub should be named;
        # need rewrite a lot...
        # for sub in self.mapsub.values:
        self.table.gen_dyn_unitclass(self.dyn_list)    #
        # need rewrite a lot...

    def print_table(self, l = 1):
        # print 1st element or each slot;
        pass


# coding: utf-8

from urllib.parse import urlencode

import qiniu
import aiohttp

from aioqiniu.services import StorageServiceMixin
from aioqiniu.exceptions import HTTPError       # noqa: F401

__version__ = "1.2.0"

class QiniuClient(StorageServiceMixin):
    """七牛云存储异步客户端"""

    def __init__(self, access_key: str, secret_key: str, httpclient=None):
        """初始化七牛云异步客户端

        :param access_key: 七牛云的AccessKey
        :param secret_key: 七牛云的SecretKey
        :param httpclient: 自定义`aiohttp.ClientSession`对象，默认为空，自动创建
        """
        self.__access_key = access_key
        self.__secret_key = secret_key
        self._auth = qiniu.Auth(access_key, secret_key)
        self._auto_close_httpclient = False
        if httpclient is None:
            httpclient = aiohttp.ClientSession()
            self._auto_close_httpclient = True
        self._httpclient = httpclient

    def get_token(self, data: str):
        """从原始数据中生成的token

        该方法等同于`qiniu.Auth.token`

        :param data: 待签名数据

        :return: 数据签名
        """
        return self._auth.token(data)

    def get_token_with_data(self, data: str):
        """生成带原始数据的token

        该方法等同于`qiniu.Auth.token_with_data`

        :param data: 待签名数据

        :return: 数据签名，含已编码的原数据
        """
        return self._auth.token_with_data(data)

    def get_access_token(self, path: str, query="", body="") -> str:
        """生成七牛云的管理凭证(access token)

        :param path: URL路径
        :param query: URL查询字符串，可以是str或dict类型，默认为空
        :param body: 请求body，默认为空

        :return: 七牛云的管理凭证(access token)

        详见：https://developer.qiniu.com/kodo/manual/1201/access-token
        """
        if not query:
            return self._auth.token("{}\n{}".format(path, body))
        if isinstance(query, dict):
            query = urlencode(query)
        return self._auth.token("{}?{}\n{}".format(path, query, body))

    def get_upload_token(self, bucket: str, key=None, expires=3600,
                         policy=None, strict_policy=True) -> str:
        """生成七牛云的上传凭证(upload token)

        :param bucket: 空间名
        :param key: 上传的文件命名，默认为空
        :param expires: 上传凭证过期时间，单位为秒，默认为3600
        :param policy: 上传策略，默认为空

        :return: 七牛云的上传凭证(upload token)

        详见：https://developer.qiniu.com/kodo/manual/1208/upload-token
        """
        return self._auth.upload_token(bucket, key, expires, policy,
                                       strict_policy)

    def get_private_download_url(self, url, expires=3600) -> str:
        """生成私有资源的下载url

        :param url: 私有资源的url
        :param expires: 下载url的过期时间，单位为秒，默认为3600

        :return: 私有资源的下载url

        详见：https://developer.qiniu.com/kodo/manual/1202/download-token
        """
        return self._auth.private_download_url(url, expires)

    def __del__(self):
        if self._auto_close_httpclient:
            self._httpclient.close()

    pass


file = open(("source.txt"), "r")  # (os.path.join(local_dir, "source.txt"), "r") as file:
read_file_to_list = file.read().split('\n') #['row 1', 'row 2', 'row 3', 'row 4', 'row 5', 'row 6', 'row 7', 'row 8', 'row 9', 'row 10']
rows_count = len(read_file_to_list) #10
recur_fibo = lambda n: n if n <= 1 else recur_fibo(n - 1) + recur_fibo(n - 2)
fibo_numbers = ([recur_fibo(i) for i in range(rows_count)]) #[0, 1, 1, 2, 3, 5, 8, 13, 21, 34]
result_reversed_string = [i[::-1] for i in read_file_to_list if read_file_to_list.index(i) + 1 in fibo_numbers] #['1 wor', '2 wor', '3 wor', '5 wor', '8 wor']
cuted_row = [(i[(i.rfind(' ') + 1):]) for i in result_reversed_string] #['wor', 'wor', 'wor', 'wor', 'wor']

with open("output.txt", "x") as newfile:
    for count, value in enumerate(cuted_row, start=1):
        newfile.write(f'{(count)} {"".join(value)}\n')

    newfile.close()
file.close()


from sikuli import *
import logging
import myTools

#---------------------------------------------------#
def fCreate_OneDiscount(pClient,pMonth,pAmount):
#---------------------------------------------------#

    logging.debug('- Create_OneDiscount: ' + str(pMonth) + "-" + pClient + " = " + str(pAmount))

    # new transaction
    type("n",KeyModifier.CTRL)
    myTools.waitForTransEntry()

    # switch to Discount

    type(Key.UP)    # this is to get by a UI defect
    time.sleep(1)

    type("d")
    time.sleep(1)
    type(Key.TAB)

    # client
    myTools.enterClient(pClient)

    # date
    tranDate = str(pMonth) + "/28/" + Settings.dataYear
    type(tranDate)
    time.sleep(1)
    type(Key.TAB)

    # Amount
    type(str(pAmount))
    type(Key.TAB)

    # Description
    type("a",KeyModifier.CTRL)
    type("Discount: " + pClient + " - " + tranDate)
    type(Key.ENTER)
    time.sleep(1)

    # move to invoice list
    if (int(Settings.tsVersion) > 2016) and (Settings.tsDB == "PREM"):
        myTools.pressTAB(2)
    else:
        myTools.pressTAB(1)

    # move to last entry
    myTools.moveToLastTrans()
    myTools.clickApplyOne()

    # save
    type("s",KeyModifier.CTRL)
    myTools.waitForTransSave()

#---------------------------------------------------#
def fCreate_Discounts(pMonth):
#---------------------------------------------------#

    myTools.sectionStartTimeStamp("discounts" + str(pMonth))
    logging.debug('fCreate_Discounts: ' + str(pMonth))

    # list the client that will get a refund each month
    discountClients = ["Natick","Orange","Oakham","Oak Bluffs","Southampton","Otis","Oxford","Leyden","Monroe","Monson","Methuen","Uxbridge"]
    oneClient = discountClients[(pMonth - 1)]

    myTools.getFocus()

    # open a/r tran list
    type("t",KeyModifier.CTRL)
    myTools.waitForTransList()

    discountAmount = 49 + pMonth/float(100)
    fCreate_OneDiscount(oneClient,pMonth,discountAmount)

    type(Key.F4,KeyModifier.CTRL)
    time.sleep(1)
    type(Key.F4,KeyModifier.CTRL)

    myTools.sectionEndTimeStamp()
    myTools.checkProcesses()


from django.contrib import auth
from django.core import exceptions, management
from django.db import connection
from django.test import TestCase

import models

from django_postgres.function import (create_function, create_functions,
    _function_exists)

class FunctionModelTestCase(TestCase):
    """Test the Function API.
    """
    def setUp(self):
        management.call_command('sync_pgfunctions', *[], **{})

    def test_get_counter(self):
        """Must run call on the manager before querying the result.
        """
        foo_user = auth.models.User.objects.create(
            username='foo', is_superuser=True)
        foo_user.set_password('blah')
        foo_user.save()

        foo_superuser = models.UserTypeCounter.objects.call(
            (True, ))

        self.assertEqual(foo_superuser.get().my_count, 1)

    def test_uncalled(self):
        """Cannot execute the statement unless you explicitly call it first
        """
        foo_user = auth.models.User.objects.create(
            username='foo', is_superuser=True)
        foo_user.set_password('blah')
        foo_user.save()

        self.assertRaises(
            exceptions.ObjectDoesNotExist,
            models.UserTypeCounter.objects.filter,
            pk=1)

class LowLeveFunctionTestCase(TestCase):
    """Low level tests for function creation.
    """
    def test_create_function(self):
        """Create a function with the low-level create_function API.
        """
        field = ('a_field integer', )
        definition = 'SELECT 1 from auth_user WHERE id = $1'
        name = 'my_function (integer)'
        created = create_function(connection, name, field, definition)

        self.assertEqual(created, 'CREATED')

    def test_update_function(self):
        """Update a function with create_function. Functions can only be
        updated if their signature matches the existing function.
        """
        field = ('a_field integer', )
        definition = 'SELECT 1 from auth_user WHERE id = $1'
        name = 'my_function (integer)'
        create_function(connection, name, field, definition)

        definition = 'SELECT 2 from auth_user WHERE id = $1'

        updated = create_function(connection, name, field, definition)

        self.assertEqual(updated, 'UPDATED')

    def test_error_function(self):
        """Error out if the user tried to update a function with an
        incompatible signature.
        """
        field = ('a_field integer', )
        definition = 'SELECT 1 from auth_user WHERE id = $1'
        name = 'my_function (integer)'
        create_function(connection, name, field, definition)

        name = 'my_function (integer, integer)'
        definition = 'SELECT 1 from auth_user WHERE id > $1 and id < $2'

        updated = create_function(connection, name, field, definition)

        self.assertEqual(updated, 'ERROR: Manually Drop This Function')

    def test_create_functions_from_models(self):
        """Create functions using the create_functions and passing the models
        module.
        """
        create_result = create_functions(models)

        for status, _, _ in create_result:
            self.assertEqual(status, 'CREATED')

        # Now check it was created
        cursor_wrapper = connection.cursor()
        cursor = cursor_wrapper.cursor
        self.assertEqual(_function_exists(cursor, 'user_type'), True)

    def test_create_command(self):
        """Test the sync_pgfunctions command.
        """
        management.call_command('sync_pgfunctions', *[], **{})

        # Check it was created
        cursor_wrapper = connection.cursor()
        cursor = cursor_wrapper.cursor
        self.assertEqual(_function_exists(cursor, 'user_type'), True)


import requests, time, json
import pandas as pd
import numpy as np
from collections import *
import matplotlib.pyplot as pl
from hmmlearn import hmm
# from SimpleHOHMM import HiddenMarkovModelBuilder as Builder
import random
import multiprocessing
from multiprocessing import Pool
import multiprocessing.pool
from sklearn.preprocessing import MinMaxScaler
from sklearn.utils import check_random_state
from sklearn.model_selection import train_test_split
import joblib
import time
from tqdm import tqdm
#import cupy as np
#import numba as nb

"""
 *  This program will be a little different from the last.
 *  Instead of sticking with normal timeseries data, I will
 *  use volume based data. What I mean by this is instead of
 *  having periods be completed after certain periods of time,
 *  periods will end after a certian amount of volume has been
 *  accumulated.
 *
 *  The idealistic benefit of making this change is that
 *  the data will better represent 'true' price movements.
 *  Price movement isn't driven by time, price movement is
 *  driven by volume. To see this, time can move forward with
 *  no volume and the price will never change because of
 *  the lack of voulme.
 *
 *  Using volume will add challenges from a programming standpoints.
 *  Where in timeseries data, periods end regularly and are determined
 *  external from the market, with volume based data, trades aren't
 *  and depending on the market being conscious of these periods sizes
 *  will be of much greater importance.
 *  Along with this, visualizing will be very important for me to make
 *  sense of the data I'm seeing. I've been finding it difficult to
 *  locate anything on this form of analysis.
 Number of bullish/bearish periods and their ratio
 Body size of bullish/bearish periods
 Number of consecutive periods
 TODO:
    See what actual returns will be if you open a potistion at the beginning of a period (when theres positive movement) then close at the end
        compare this method to closing after with certain "high" percentages are reaches.
        then compare this to adding a stop loss
    GPU computations
"""

#Load credentials from json
#cred = json.load(open("credentials.json"))
files = json.load(open("files.json"))

def readFiles():
    #da = pd.read_csv(''+files['BTC']+'/2013/merged.csv')
    #da.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    #dat = pd.read_csv(''+files['BTC']+'/2014/merged.csv')
    #dat.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    #data = pd.read_csv(''+files['BTC']+'/2015/merged.csv')
    #data.columns = ['time', 'open', 'close', 'high', 'min', 'volume']

    #sticking to more recent data
    data0 = pd.read_csv(''+files['BTC']+'/2016/merged.csv')
    data0.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    data1 = pd.read_csv(''+files['BTC']+'/2017/merged.csv')
    data1.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    data2 = pd.read_csv(''+files['BTC']+'/2018/merged.csv')
    data2.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    #print(sum(pd.concat([da, dat, data, data0, data1, data2], ignore_index=True)['close']))
    return pd.concat([data0, data1, data2], ignore_index=True)

def readTestFiles():
    data = pd.read_csv(''+files['BTC']+'/2019/merged.csv')
    data.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    return pd.concat([data], ignore_index=True)

def readTestFiles():
    data = pd.read_csv(''+files['BTC']+'/2019/merged.csv')
    data.columns = ['time', 'open', 'close', 'high', 'min', 'volume']
    return pd.concat([data], ignore_index=True)

def readTestModelFiles():
    dat = pd.read_csv(''+files['AMD']+'/AMD_2000_2009.csv')
    dat.columns = ['time', 'open', 'high', 'min', 'close', 'volume']
    #dat['time'] =  pd.to_datetime(dat['time'], infer_datetime_format=True)
    #dat = dat.set_index('time')

    data = pd.read_csv(''+files['AMD']+'/AMD_2010_2019.csv')
    data.columns = ['time', 'open', 'high', 'min', 'close', 'volume']
    #data['time'] =  pd.to_datetime(data['time'], infer_datetime_format=True)
    #data = data.set_index('time')

    #print(data)
    return pd.concat([dat, data])

def readRecentTestModelFiles():
    data = pd.read_csv(''+files['AMD']+'/AMD_2010_2019.csv')
    data.columns = ['time', 'open', 'high', 'min', 'close', 'volume']
    #data['time'] =  pd.to_datetime(data['time'], infer_datetime_format=True)
    #data = data.set_index('time')

    #print(data)
    return pd.concat([data])

def readSecondData(startY, startM, startD):
    dataPoints = []
    for i in range(startY, 2021):
        for j in range(startM, 13):
            for k in range(startD, 32):
                #print(''+files['AMD-S'] + str(i) + str(j).zfill(2) + str(k).zfill(2)+'/amd.csv')
                #dataPoints.append(pd.read_csv(''+files['AMD-S'] + str(i) + str(j).zfill(2) + str(k).zfill(2)+'/amd.csv'))
                #print(i)
                try:
                    dataPoints.append(pd.read_csv(''+files['AMD-S'] + str(i) + str(j).zfill(2) + str(k).zfill(2)+'/amd.csv'))
                    dataPoints[-1].columns = ['time', 'open', 'high', 'min', 'close', 'volume', 'susp']
                    dataPoints[-1]['time'] = dataPoints[-1]['time'].apply(lambda x: pd.to_datetime(''+str(i)+"-"+ str(j).zfill(2) + "-" + str(k).zfill(2)) + np.timedelta64(x, 'ms'))
                    dataPoints[-1] = dataPoints[-1].set_index('time')
                    dataPoints[-1] = dataPoints[-1].between_time('8:29:59', '15:00:00')
                    dataPoints[-1]['volume'] = np.log(dataPoints[-1]['volume'])
                    #print(dataPoints[-1])
                except Exception:
                    continue

    return dataPoints
def readModel():
    data0 = pd.read_csv(''+ files['local']+'/Model-V1.csv')
    data0.columns = ['index', 'fracC', 'fracH', 'fracL']
    return np.column_stack((data0['fracC'], data0['fracH'], data0['fracL']))

# FLAW: If there is a minute tiemframe with enough volume to fill multiple periods, it will carry over into other periods.
# Test fix 1: s = period % dif. This will accept the large volitility, but not have it carry over into other periods
# True fix: get smaller timeframes... second based data maybe?
def getHistorical(period, data):
    hist = []
    conv = []
    o = -1
    vals = [o, -1, float('inf'), -1]
    s = 0
    dupeCount = 0
    for ind, p in data.iterrows():
        # get the open, close, min, and max for each volume period given the minute based data.
        if(vals[0] == -1 or vals[0] == 0):
            vals[0] = p['open']
        vals = [vals[0], -1, min(vals[2], p['min']), max(vals[3], p['high'])]
        s += p['volume']
        if(s > 2*period):
            dupeCount += 1
        if (s >= period):

            dif = s - period
            vals[1] = p['close']
            hist.append(vals)
            if(dif!=0):
                o = p['close']
                s = period % dif
            else:
                o = -1
                s = 0
            vals = [o, -1, float('inf'), -1]

    # Make sure to catch the last data point, even if it isn't full.
    if(not (vals[1] == -1)):
        hist.append(vals)

    #print(str(dupeCount) + " condensed pointes for period " + str(period))
    #print(str(len(hist)) + " number of points for period " + str(period))
    hist = pd.DataFrame(hist, columns = ['open', 'close', 'min', 'max'])
    return (hist, period)

"""
Calcuate the EMA of a given dataset
"""
def ema(length, data):
    return data.ewm(span=length, adjust=False, min_periods=length-1).mean()

    #return (hist, period)

"""
Tag data as either bullish or bearish
Length - Length of EMA to lag
Data - Dataframe
"""
def tagData(length, data):
    EMA = ema(length, data['close'])
    slope = EMA.diff().copy()      # rough estimate
    #tagged = data.copy()
    data = data.copy()
    data['tag'] = pd.Series([0 for x in range(data['close'].__len__())], index=data.index)

    condition = slope.loc[slope > 0].index
    data.loc[condition, ['tag']] = 1

    return data

"""
Tag data as either bullish or bearish
minSize - Min size of success for dataframe to accept
Data - Dataframe
Vers - String to identify in parallelism.
"""
def splitData(minSize, data, vers):
    split = []
    indP = data.index.tolist()[0]
    s = False
    for ind, row in tqdm(data.iterrows(), desc = vers + " Splitting"):
        d = (ind - indP)/np.timedelta64(1, 'm')
        if(d < .03):
            if(not s):
                split.append(pd.DataFrame())
                split[-1] = split[-1].append(row)
                s = True
            else:
                split[-1] = split[-1].append(row)

        else:
            s = False
            if(len(split)>0 and split[-1].__len__() < minSize):
                del split[-1]
        indP = ind
    return split

"""
    converts from open, close, min, max
    to period change, max/min, fractional high, and fractional low
    TODO: DEBUGGGGGGGGG (DATA PRESENTATION IS THE MOST IMPORTANT PART OF THIS PROCESS)
"""
def convert(hist):
    #print("Converting data")
    conv = []

    o = np.array(hist['open'])
    c = np.array(hist['close'])
    h = np.array(hist['max'])
    l = np.array(hist['min'])

    fracC = []
    fracH = []
    fracL = []

    for i in range(len(o.tolist())):
        if(c[i]-o[i] < 0):
            if((o[i]-c[i])/o[i] >= 1 and (o[i]-c[i])/o[i] <=1.5):
                fracC.append(-.75)
            elif((o[i]-c[i])/o[i] > 1.5):
                fracC.append(-1)
            else:
                fracC.append(1/np.log((o[i]-c[i])/o[i]))
        elif(c[i]-o[i] > 0):
            if((c[i]-o[i])/o[i] >= 1 and (c[i]-o[i])/o[i] <= 1.5):
                fracC.append(.75)
            elif((c[i]-o[i])/o[i] > 1.5):
                fracC.append(1)
            else:
                fracC.append(-1/np.log((c[i]-o[i])/o[i]))
        else:
            fracC.append(0)

        #upward movements are unbound. should consider a way to account for this.
        if((h[i]-o[i]) <= 0):
            fracH.append(0)
        elif(np.log((h[i]-o[i])/o[i]) >= 0):
            fracH.append(10)
        else:
            fracH.append(-1/np.log((h[i]-o[i])/o[i]))

        #l is bound by zero
        if((o[i]-l[i]) <= 0):
            fracL.append(0)
        elif(np.log((o[i]-l[i])/o[i]) == 0):
            fracL.append(10)
        else:
            fracL.append(-1/np.log((o[i]-l[i])/o[i]))

    scalar = MinMaxScaler()
    scalar.fit(np.array(fracC).reshape(-1, 1))

    # Standardize data to a normal distribution centered around 0
    # Make price movements more priminent AND more consistent
    # Hopefully will increase the effectiveness of the model and its training.

    # works better with smaller non-volumetric periods (theory. will test tonight)
    #fracC = (scalar.transform(np.array(fracC).reshape(-1, 1)).flatten()-.5)
    #fracH = (scalar.transform(np.array(fracH).reshape(-1, 1)).flatten()-.5)
    #fracL = (scalar.transform(np.array(fracL).reshape(-1, 1)).flatten()-.5)

    #print(fracC)

    return np.column_stack((fracC, fracH, fracL))

def scale(array):
    scalarH = MinMaxScaler()
    scalarH.fit(np.array(array[:,1]).reshape(-1, 1))
    fracH = (scalarH.transform(np.array(array[:,1]).reshape(-1, 1)).flatten()-.5)
    scalarL = MinMaxScaler()
    scalarL.fit(np.array(array[:,2]).reshape(-1, 1))
    fracL = (scalarL.transform(np.array(array[:,2]).reshape(-1, 1)).flatten()-.5)
    scalarC = MinMaxScaler()
    scalarC.fit(np.array(array[:,0]).reshape(-1, 1))
    fracC = (scalarC.transform(np.array(array[:,0]).reshape(-1, 1)).flatten()-.5)
    return (np.column_stack((fracC, fracH, fracL)), (scalarC, scalarH, scalarL))

def run(period):
    #print('getting historical')
    #hist = getHistorical(period, readFiles())[0]
    #print('getting historical test')

    testFiles = readTestModelFiles()
    testFiles['time'] =  pd.to_datetime(testFiles['time'], infer_datetime_format=True)
    testFiles = testFiles.set_index('time').loc['1/1/2018':'1/1/2019']
    print(testFiles)

    vol = int(testFiles['volume'].sum())

    print(vol)

    histT = getHistorical(period, testFiles)[0]

    #conv = convert(hist)

    #hist.to_csv('models/Hist-V1.csv')
    #histT.to_csv('models/Test-V1.csv')
    #pd.DataFrame(conv).to_csv('models/Model-V1.csv')

    #for i in conv:
    #    print(i)

    #-------------------------------------------------------------------------------------------------------------------

    print('make hmm')

    HMM = hmm.GaussianHMM(n_components = 11 , covariance_type="full", random_state=7, n_iter = 1000)

    HMM.fit(readModel())
    print(HMM.sample(10))
    print(HMM.transmat_)
    print('complete')

    #-------------------------------------------------------------------------------------------------------------------
    scores  = defaultdict(list)
    pSize = random.randint(10, 75)
    strt = random.randint(8, histT.__len__()-pSize)
    for j in range(15):
        pSize = random.randint(10, 75)

        for i in range(75):
            #if(i == 0 and not scores[pSize] == None):
            #    break
            strt = random.randint(6, histT.__len__()-pSize)
            pred, sc, ret = predict(HMM, histT, strt, strt+pSize, 5, 25000, False)
            scores[pSize].append((pred, sc, ret))

    #-------------------------------------------------------------------------------------------------------------------

    predictedCloseForTest, _, _ = predict(HMM, histT, strt, strt+pSize, 3, 25000, True)
    trueOpenForTest       = histT.iloc[strt:strt+pSize]['open'].values
    trueCloseForTest      = histT.iloc[strt:strt+pSize]['close'].values

    print("50 random periods w/50 different random tests resuts::")

    for i in scores.keys():
        s = str(sum(n for _, n, _ in scores[i])/len(scores[i]))[0:5]
        ret = str(sum(n for _, _, n in scores[i])/len(scores[i]))[0:5]
        print("For the 75 random tests over " + str(i) + " periods, the HMM determined the direction correctly: " + s + "% of the time. Ret: " + ret)
        #plotter(trueCloseForTest, predictedCloseForTest, trueOpenForTest, )

class NoDaemonProcess(multiprocessing.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)

# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool
# because the latter is only a wrapper function, not a proper class.
class MyPool(multiprocessing.pool.Pool):
    Process = NoDaemonProcess

def optimizeGen():
    # Optimse the volumetric period for a given stock on the given model.
    # This will be a test to see if we can use 'generic' models as I theorize we can.

    scores = []
    testFiles = readTestModelFiles()
    testFiles['time'] =  pd.to_datetime(testFiles['time'], infer_datetime_format=True)
    testFiles = testFiles.set_index('time').last('1Y')
    print(testFiles)
    vol = int(testFiles['volume'].sum())
    model = readModel()
    HMM = hmm.GaussianHMM(n_components = 11, covariance_type="full", random_state=7, n_iter = 1000)
    HMM.fit(model)
    for i in tqdm(range(vol//365//6, vol//92, vol//365//4)):
        his = []
        res = []

        with Pool() as p:
            his = p.starmap(getHistorical, [(x, testFiles) for x in range(i, i + vol//365//4 - 10, ( vol//365//4)//4)])

        with Pool() as p:
            res = p.starmap(runTests, [(HMM, j[0], 15, 75, 5, j[1], -1) for j in his])

        for j in res:
            s = 0
            for k in j[0].keys():
                s+=sum(j[0][k])/len(j[0][k])
                t = k

            s = s/len(j[0].keys())
            scores.append((j[1], s))

        scores.sort(key = lambda x: x[1], reverse = True)
        print(scores[0:5])
    return scores

"""
Data - Trainging Dataframe
DataT - Testing Datafram
Mult - Multiplier from smallest accepted timeframe (will change if smaller data sizes can be acquired) (SECOND DATA ON THE COMEUP??)
vers - String to identify in parallelism
"""
def optimize(data, dataT, mult, vers):
    # Dictionary from Period to dict from HMM components to dict from HMM lookBack size to list of tuples of test length and score
    optimizer = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list))))

    v  = sum([x['volume'].sum() for x in data])//sum([len(x.index) for x in data])       # Average vol/min across all training periods.
    #v = data[0]['volume'].sum() // data[0].index.normalize().unique().__len__()         # Average vol/min on most recent (and representative) dataset
    vtr = [x['volume'].sum()//len(x.index) for x in data]       # Average Vol/Min in each given train period
    vT = [x['volume'].sum()//len(x.index) for x in dataT]       # Average Vol/min in each given test period

    volAdjust  = [x/v for x in vtr]                             # Attempt to standardize the avg vol per min throughout training data.
    volAdjustT = [x/v for x in vT]                              # same thing for test data.

    vol = sum([x['volume'].sum() for x in data])                # Total Volume across all periods

    dataPoints = sum([len(x.index) for x in data])              # Total number of datapoints in training
    dataPointsT = sum([len(x.index) for x in dataT])            # Total number of datapoints in testing

    # Document all acquired info above for potential debugging purposes
    with open(vers+'.txt', 'w') as f:
        f.write(vers+'\n')
        f.write('Volume: ' + str(vol)+ '\nTrain Data points: ' + str(dataPoints) + '\nTest Data Points: '+str(dataPointsT) + '\n')
        f.write('Vol/Min: ' + str(vol/dataPoints) + '\n')
        #f.write("Train AdjVol: " + str(volAdjust) + '\n')
        #f.write("Test  AdjVol: " + str(volAdjustT) + '\n')

    averageBest50 = []
    for i in tqdm(range(int(vol/(dataPoints/mult)), int(vol/(dataPoints/8)), int(vol/(dataPoints/mult*2))), desc=vers+" volume Progress"):
        print() #add some spacing and stuff...
        hist = [convert(getHistorical(i*volAdjust[x], data[x])[0]) for x in range(len(data))]

        # keep things spicy ;)
        random.shuffle(hist)

        # Keep all datapoints seperate@@@@@@@
        histT = [getHistorical(i*volAdjustT[j], dataT[j])[0] for j in range(len(dataT))]
        converted, scalar = scale(np.concatenate(hist))
        joblib.dump(scalar, "scales/"+vers+'-'+str(i)+".pkl")
        for j in tqdm(range(2), desc=vers+" Components Progress"):

            res = []
            with Pool() as p:
                res = p.starmap(makeModel, [(j*3 + x, converted, scalar, hist, histT, vol, i, dataPoints, vers) for x in range(3, 6)])

            optimizer[i][j*3][1] = res[0]
            optimizer[i][j*3+1][1] = res[1]
            optimizer[i][j*3+2][1] = res[2]

        for j in optimizer[i].keys():
            for k in optimizer[i][j].keys():
                s = 0
                for l in optimizer[i][j][k][0].keys():
                    s += sum(optimizer[i][j][k][0][l])/len(optimizer[i][j][k][0][l])

                sc = s/len(optimizer[i][j][k][0].keys())

                if len(averageBest50) == 0 or averageBest50[-1][3] < sc:
                    averageBest50.append((i, j+3, k, sc))
                    averageBest50.sort(key = lambda x: x[3], reverse=True)
                    if len(averageBest50) > 50:
                        averageBest50.pop()
        with open(vers+'.txt', 'a') as f:
            f.write(vers + " ::: " + str(averageBest50[0:5])+"\n")

    # Log all results
    for i in optimizer.keys():
        for j in optimizer[i].keys():
            for k in optimizer[i][j].keys():
                s = 0
                for l in optimizer[i][j][k][0].keys():
                    s += sum(optimizer[i][j][k][0][l])/len(optimizer[i][j][k][0][l])
                sc = s/len(optimizer[i][j][k][0].keys())

                with open(vers+'.txt', 'a') as f:
                    f.write(vers+" : " + str(i) + " : " + str(j+3) + " : " + str(k) + " : " + str(sc) + " : " + str(optimizer[i][j][k][3]) +  " : " + str(optimizer[i][j][k][2])+"\n")

    return averageBest50
def makeModel(components, converted, scalar, hist, histT, vol, i, dataPoints, vers):
    HMM = hmm.GaussianHMM(n_components = components, covariance_type="full", n_iter = 750, verbose = False)
    HMM.fit(converted, lengths = [x.__len__() for x in hist])
    joblib.dump(HMM, "models/"+vers+str(i) + "-" + str(components) + "-" + str(1)+".pkl")
    return runTests(HMM, histT, 250, 1, vol, i, dataPoints, vers + str(i) + "-" + str(components) + "-" + str(1), scalar)

def runTests(HMM, histT, iter1, lookBack, v, p, days, vers, scalar):
    scores  = defaultdict(list)
    strt = 0
    f = 0
    avgRet = 0
    for j in range(iter1):
        # Pick random histT period
        randInd = random.randint(0, len(histT)-1)

        if(len(histT[randInd]) > lookBack):
            pred, sc, ret = predict(HMM, histT[randInd], lookBack, 25000, False, vers, scalar)
            scores[randInd].append(sc)
            avgRet += ret
        else:
            f += 1
    return (scores, p, f, avgRet/(iter1-f))

# Need to smooth data out before hand so i can get all datapoints in a given period. But if this is accurate, might just need to have the model make
# less decisions that are more accurate.
def predict(hmm, histT, lookBack, ret, plot, vers, scalar):
    pred = []

    for i in range(lookBack, len(histT.index)):
        oPrice = histT.iloc[i]['open']
        cPrice = histT.iloc[i]['close']

        prevD = histT.iloc[i-lookBack:i]

        conv = convert(prevD)
        conv = np.column_stack(((scalar[0].transform(np.array(conv[:,0]).reshape(-1, 1)).flatten()-.5), (scalar[1].transform(np.array(conv[:,1]).reshape(-1, 1)).flatten()-.5), (scalar[2].transform(np.array(conv[:,2]).reshape(-1, 1)).flatten()-.5)))
        stateSeq = hmm.predict(conv)
        # print(vers + " - " + str(stateSeq))

        randstate = check_random_state(hmm.random_state)
        #print(vers + " - " + str(randstate.get_state()))
        nextState = (np.cumsum(hmm.transmat_, axis=1)[stateSeq[-1]] > randstate.rand())
        # print(np.cumsum(hmm.transmat_, axis=1)[stateSeq[-1]])
        # #print(vers + " - " + str(randstate.rand()))
        # print(vers + " - " + str(nextState))
        # print(vers + " - " + str(nextState.argmax()))
        nextObs = hmm._generate_sample_from_state(nextState.argmax(),randstate)
        # print(vers + "----------------------------------")
        #print(str(nextObs[0]) + " - " + vers)
        # if(nextObs[0] > 0):
        #     pred.append(oPrice / (np.exp(1.0/nextObs[0])) + oPrice)
        # elif(nextObs[0] < 0):
        #     pred.append(-oPrice / (np.exp(-1.0/nextObs[0])) + oPrice)
        # else:
        #     pred.append(oPrice)
        pred.append(oPrice * (1+nextObs[0]*.005))

    # Score model/results (Compare predictions to actual results)
    c = 0
    s = 0
    v = 0
    for i in histT.iloc[lookBack:]['open'].values:
        if not ret == -1:
            if (vers[:4]=="BULL"):
                if(pred[s]-i > 0):
                    temp = ret*.1
                    ret -= temp
                    ret += (temp) * histT.iloc[s+lookBack]['close']/i
            else:
                if(pred[s]-i < 0):
                    temp = ret*.1
                    ret -= temp
                    ret += (temp) * i/histT.iloc[s+lookBack]['close']
        if (vers[:4]=="BULL"):
            if((pred[s]-i)>0 and (histT.iloc[s+lookBack]['close']-i) > 0):
                c+=1
            if(pred[s]-i > 0):
                v+=1
        else:
            if((pred[s]-i)<0 and (histT.iloc[s+lookBack]['close']-i) < 0):
                c+=1
            if(pred[s]-i < 0):
                v+=1
        s+=1

    #print("for this sample, the HMM predicted the correct direction " + str(100*(c/s)) + "% of the time. P = " + str(endInd-startInd) + ".")

    if(plot):
        # only log 10% of plots to save time and memory
        rand = random.random()
        if(rand < .1):
            plotter(histT.iloc[lookBack:]['close'].values, pred,histT.iloc[lookBack:]['open'].values, vers+"-"+str(ret)[0: 5])
    if(v == 0):
        c = 1
        v = 2
    return pred, (100*(c/v)), ret

def plotter(data, dataP, dataPO, name):
    #print(data)
    #print(dataP)
    pl.style.use('ggplot')
    plot = pl.figure()
    axis = plot.add_subplot(111)
    axis.plot([x for x in range(data.__len__())], data, 'bo-', label='real close')
    axis.plot([x for x in range(data.__len__())], dataP, 'r+-', label='predicted close (based on realO)')
    axis.plot([x for x in range(data.__len__())], dataPO, 'b+-', label='real open')
    pl.legend()
    pl.savefig("D:/plots/"+name+".png")
    pl.close(plot)

def start():
    #t = api.get_clock()

    # PERIODS::: (average 12 ticks a day)
    #   XRP: 1934152
    #   XRP: 23209824 (average of a day)
    #   ETH: 248185   (works really well.) 11 - 3 - 60%
    #   BTC: 27040    (should mirror ETH)  11 - 5 - 65%      - Basis for general model
    #   AMD  129376990                     11 - 5 - 59.64%   - First result from general model

    # Stuff for minute based data
    #-------------------------------------------------------------------------------
    #data = readTestModelFiles()
    #data['time'] =  pd.to_datetime(data['time'], infer_datetime_format=True)
    #data = data.set_index('time')
    #volcond = data.loc[data['volume'] == 0].index
    #data.loc[volcond, 'volume'] = 1
    #data['volume'] = np.log(data['volume'])

    #print(data['volume'])
    #print(np.log(0))

    # data = data.iloc[len(data['volume'].index)//2:]
    #data = data.iloc[5000:10000]
    # testtag = tagData(50, data.iloc[5000:10000])
    # testcond = testtag.loc[testtag['tag'] == 0] .index
    # testtag = testtag.loc[testcond]
    # testd = splitData(45, testtag, 'test')
    # test = getHistorical(80000, testd[0])[0]
    # convert(test)
    # print(test.iloc[0:50])
    # tagged = tagData(50, data)
    #-------------------------------------------------------------------------------

    # Automatically tag data as bearish or bullish with a simple EMA
    #   (keeping the tagging simple will provide proof of concept.
    #    more technical indicators can be used in the future)
    # This will maintain consistency in training data.

    # data = readSecondData(2018, 1, 1)
    # print(data)
    # tagged = [tagData(40, x) for x in data]
    # BearTag = []
    # BullTag = []

    # # get each conditional for each dataset
    # conditionBear = [tag.loc[tag['tag'] == 0].index for tag in tagged]
    # conditionBull = [tag.loc[tag['tag'] == 1].index for tag in tagged]

    # BearTag = pd.concat([data[i].loc[conditionBear[i]] for i in range(len(data))])
    # BullTag = pd.concat([data[i].loc[conditionBull[i]] for i in range(len(data))])

    # ############################################################################
    # # conditionBear = tagged.loc[tagged['tag'] == 0].index
    # # conditionBull = tagged.loc[tagged['tag'] == 1].index
    # # BearTag = pd.concat([d.loc[conditionBear] for d in data], copy = False)
    # # BullTag = pd.concat([d.loc[conditionBear] for d in data], copy = False)
    # ############################################################################

    # print(BearTag)
    # joblib.dump(BearTag, "data/BearTag40.pkl")
    # joblib.dump(BullTag, "data/BullTag40.pkl")
    res = []
    # Split data into continous sections
    # with MyPool(2) as p:
    #    res = p.starmap(splitData, [(40, BullTag, 'Bull'), (40, BearTag, 'Bear')])

    # joblib.dump(res, "data/split40.pkl")
    res = joblib.load('data/split20.pkl')
    # Log lost datapoints
    #print("Split:: " + str(sum([len(x.index) for x in res[0]])))
    #print("Original:: " + str(len(BearTag.index)))

    random.shuffle(res[0])
    random.shuffle(res[1])
    Bull, BullT = train_test_split(res[0], train_size = .75, shuffle = False)
    Bear, BearT = train_test_split(res[1], train_size = .75, shuffle = False)

    # TRY LOG SHIFTING THE VOLUME BECAUSE IT IS SO RIGHTLY SKEWED...

    with MyPool(2) as p:
         res = p.starmap(optimize, [(Bull, BullT, 2, "BULL"),
                                    (Bear, BearT, 2, "BEAR")])

    # with MyPool(2) as p:
    #     res = p.starmap(optimize, [([data.loc['10/8/2019':'12/31/2019'], data.loc['5/1/2018':'9/17/2018'], data.loc['1/1/2019':'8/1/2019'], data.loc['1/1/2009':'1/1/2010'], data.loc['5/5/2005':'2/8/2006']],
    #                                 [data.loc['2/2/2016':'5/1/2017'], data.loc['3/1/2009':'1/1/2010']], .005, "BULL"),
    #                                ([data.loc['8/12/19':'10/4/19'], data.loc['9/2/18':'12/26/18'], data.loc['2/2/18':'4/20/18'], data.loc['4/11/12':'12/13/12'], data.loc['3/3/06':'8/16/06'], data.loc['10/4/06':'5/24/07'], data.loc['11/1/07':'5/2/08']],
    #                                 [data.loc['2/27/18':'4/17/18'], data.loc['5/19/11':'12/21/11'], data.loc['5/4/10':'9/1/10'], data.loc['9/10/08':'3/4/09'], data.loc['9/4/14':'10/5/15']], .005, "BEAR")])

"""
 ToDo: Run tests again and choose random periods to plot for the highest scoring tests. Make sure the program isnt just a fancy EMA trader (or one that doesnt use Machine learning to its true potential)...
       If it does just continously buy until the ema flips, try re-weighing what the program consideres success. (For example, Extra points for correctly determining when the next period wont be positive)
"""

if __name__ == "__main__":

    #print(optimize())
    #print(optimizeGen())
    start()

"""
Moved to seperate file
"""


import os
import numpy as np
np.random.seed(101)

def load_synthetic(m, n, alpha):
    # Reference: https://github.com/andylamp/federated_pca/blob/master/synthetic_data_gen.m
    k = min(m, n)
    U, _ = np.linalg.qr(np.random.randn(m, m))
    Sigma = np.array(list(range(1, k+1))).astype(np.float32) ** -alpha
    V = np.random.randn(k, n)
    Y = (U @ np.diag(Sigma) @ V) / np.sqrt(n-1)
    yn = np.max(np.sqrt(np.sum(Y ** 2, axis=1)))
    Y /= yn
    return Y, None


import numpy as np
import sympy
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from pylab import figure, axes, pie, title, show

print('Graphik1')
print('Steigung','Y-Achsenabschnitt')
x, y = np.loadtxt('reichweite20.txt', unpack=True,delimiter=',')
T=np.sqrt(y)

plt.plot(x, y, "kx", label="Messwerte")
plt.errorbar(x, y, yerr=T, fmt="none", capsize=3, capthick=1, ms=9, markerfacecolor="red")

y=42750+0*x
plt.plot(y,label=r'$\frac{N_{0}}{2}$')

print('Erste Grade')
c, v = np.loadtxt('fit20.txt', unpack=True,delimiter=',')
def f(c,a,b):
    return a*c+b
popt, pcov = curve_fit(f, c, v)
print(popt)
print(np.diag(pcov))
c_new = np.linspace(x[10], x[-1], 500)
#
#
#
#print('Zweite Grade')
#q, w = np.loadtxt('beta3.txt', unpack=True,delimiter=',')
#def g(q,r,s):
#    return r*q+s
#pqpt, pcqv = curve_fit(g, q, w)
#print(pqpt)
#print(np.diag(pcqv))
#q_new = np.linspace(x[4], x[-1], 500)

plt.figure(1)
#plt.plot(x,y,'x')
plt.plot(c_new,f(c_new,*popt),'-', label='Lineare Regression')
#plt.plot(q_new,g(q_new,*pqpt),'-', label='Lineare Regression Hintergrundstrahlung')
plt.xlabel('Effektive Länge $x/ 10^{-3}m$')
plt.ylabel('Zählrate $N$')
plt.grid()
plt.legend()

plt.savefig('reichweite20.pdf')
print ('Fertig')


from JumpScale9 import j

import logging
from .Handlers import *

from .JSLogger import JSLogger
from .JSLoggerDefault import JSLoggerDefault
import sys

class LoggerFactory():

    def __init__(self):
        self.__jslocation__ = "j.core.logger"
        self.logger_name = 'j'
        self.handlers = Handlers()
        self.loggers = {}
        self.exclude = []

        self._default = JSLoggerDefault("default")

        self.logger = JSLogger("logger")
        self.logger.addHandler(self.handlers.consoleHandler)

        self.enabled = True
        self.filter = ["*"]  # default filter to see which loggers will be attached needs to have * or j.sal... inside

        # self.logger.debug("started logger factory")

    def _getName(self, name):

        name = name.strip().lower()

        if name == "":
            path, ln, name, info = logging.root.findCaller()
            if path.startswith(j.dirs.LIBDIR):
                path = path.lstrip(j.dirs.LIBDIR)
                name = path.replace(os.sep, '.')

        if not name.startswith(self.logger_name):
            name = "%s.%s" % (self.logger_name, name)

        if len(name) > 22:
            name = name[-22:]

        return name

    def get(self, name="", force=False):  # -> JSLogger:
        """
        Return a logger with the given name. Name will be prepend with 'j.' so
        every logger return by this function is a child of the jumpscale root logger 'j'

        """
        name = self._getName(name)

        def check_(name):
            # print("check %s"%name)
            for item in self.exclude:
                # print("check exclude:%s"%item)
                if item == "*":
                    # print("exclude %s:%s" % (item, name))
                    return False
                if name.find(item) != -1:
                    # print("exclude %s:%s" % (item, name))
                    return False
            for item in self.filter:
                # print("check include:%s"%item)
                if item == "*":
                    # print("include: %s:%s" % (item, name))
                    return True
                if name.find(item) != -1:
                    # print("include: %s:%s" % (item, name))
                    return True
            return False

        if force == False and self.enabled is False:
            self.loggers[name] = self._default
            # print("DEFAULT LOGGER (disabledlogger):%s" % name)
        else:
            if force or check_(name):
                # print("JSLOGGER:%s" % name)
                # logger = logging.getLogger(name)
                logger = JSLogger(name)
                logger.level = j.core.state.configGetFromDict("logging", "level", 'DEBUG')

                for handler in self.handlers._all:
                    logger.handlers = []
                    logger.addHandler(handler)

                self.loggers[name] = logger
            else:
                # print("DEFAULT LOGGER:%s" % name)
                self.loggers[name] = self._default

        return self.loggers[name]

    def disable(self):
        """
        will transform all loggers to empty loggers which only act on errors, but ignore logs
        """
        if self.enabled:
            self.enabled = False
            self.filter = []

            # for key, logger in self.loggers.items():
            #     # print("disable logger: %s"%key)
            #     logger.setLevel(20)
            j.application.debug = False

            self.logger_filters_add()

    def enable(self):
        """
        """
        if self.enabled is False:
            self.enabled = True
            self.filter = []
            self.init()

    # def set_quiet(self, quiet):
    #     self._quiet = quiet

    # def set_mode(self, mode):
    #     if isinstance(mode, str):
    #         if mode in _name_to_mode:
    #             mode = _name_to_mode[mode]
    #         else:
    #             raise j.exceptions.Input("mode %s doesn't exist" % mode)

    #     if mode == self.PRODUCTION:
    #         self._enable_production_mode()
    #     elif mode == self.DEV:
    #         self._enable_dev_mode()

    # def set_level(self, level=10):
    #     """
    #     Set logging levels on all loggers and handlers
    #     Added to support backward compatability
    #     """
    #     self.loggers_level_set(level=level)

    def handlers_level_set(self, level=10):
        """

        sets level in all handlers

        10=debug
        20=info

        info see:
        https://docs.python.org/3/library/logging.html#levels

        """
        for handler in self.handlers._all:
            handler.setLevel(level)

    def loggers_level_set(self, level='DEBUG'):
        """

        sets level in all handlers & loggers

        10=debug
        20=info

        info see:
        https://docs.python.org/3/library/logging.html#levels

        """
        for key, logger in self.loggers.items():
            logger.setLevel(level)
        self.handlers_level_set(level)

    def handlers_attach(self):
        """
        walk over all loggers, attach the handlers
        """
        for key, logger in self.loggers.items():
            for handler in self.handlers._all:
                logger.handlers = []
                logger.addHandler(handler)

    def memhandler_enable(self):
        # self.logger.propagate = True
        self.logger.addHandler(self.handlers.memoryHandler)

    def consolehandler_enable(self):
        # self.logger.propagate = True
        self.logger.addHandler(self.handlers.consoleHandler)

    def telegramhandler_enable(self, client, chat_id):
        """
        Enable a telegram handler to forward logs to a telegram group.
        @param client: A jumpscale telegram_bot client
        @param chat_id: Telegram chat id to which logs need to be forwarded
        """
        self.logger.addHandler(self.handlers.telegramHandler(client, chat_id))

    def handlers_reset(self):
        self.logger.handlers = []

    def logger_filters_get(self):
        return j.core.state.config_js["logging"]["filter"]

    def logger_filters_add(self, items=[], exclude=[], level=10, save=False):
        """
        items is list or string e.g. prefab, exec
        will add the filters to the logger and save it in the config file

        """
        items = j.data.types.list.fromString(items)
        exclude = j.data.types.list.fromString(exclude)
        if save:
            new = False
            for item in items:
                if item not in j.core.state.config_js["logging"]["filter"]:
                    j.core.state.config_js["logging"]["filter"].append(item)
                    new = True
            for item in exclude:
                if item not in j.core.state.config_js["logging"]["exclude"]:
                    j.core.state.config_js["logging"]["exclude"].append(item)
                    new = True
            if new:
                j.core.state.configSave()
                self.init()

        for item in items:
            item = item.strip().lower()
            if item not in self.filter:
                self.filter.append(item)

        for item in exclude:
            item = item.strip().lower()
            if item not in self.exclude:
                self.exclude.append(item)

        self.logger.debug("start re-init for logging")

        self.handlers_level_set(level)

        # make sure all loggers are empty again
        j.dirs._logger = None
        j.core.platformtype._logger = None
        j.core.state._logger = None
        j.core.dirs._logger = None
        j.core.application._logger = None
        for cat in [j.data, j.clients, j.tools, j.sal]:
            for key, item in cat.__dict__.items():
                if item is not None:
                    # if hasattr(item, '__jslocation__'):
                    #     print (item.__jslocation__)
                    if 'logger' in item.__dict__:
                        item.__dict__["logger"] = self.get(item.__jslocation__)
                    item._logger = None
        self.loggers = {}

        # print(j.tools.jsloader._logger)
        # print(j.tools.jsloader.logger)

    def init(self):
        """
        get info from config file & make sure all logging is done properly
        """
        self.enabled = j.core.state.configGetFromDict("logging", "enabled", True)
        level = j.core.state.configGetFromDict("logging", "level", 'DEBUG')
        self.loggers_level_set(level)
        self.handlers_level_set(level)
        self.filter = []
        self.loggers = {}
        items = j.core.state.configGetFromDict("logging", "filter", [])
        exclude = j.core.state.configGetFromDict("logging", "exclude", [])
        self.logger_filters_add(items=items, exclude=exclude, save=False)

    # def enableConsoleMemHandler(self):
    #     self.logger.handlers = []
    #     # self.logger.propagate = True
    #     self.logger.addHandler(self.handlers.memoryHandler)
    #     self.logger.addHandler(self.handlers.consoleHandler)

    # def _enable_production_mode(self):
    #     self.logger.handlers = []
    #     self.logger.addHandler(logging.NullHandler())
    #     # self.logger.propagate = True

    # def _enable_dev_mode(self):
    #     logging.setLoggerClass(JSLogger)
    #     self.logger.setLevel(logging.DEBUG)
    #     self.logger.propagate = False
    #     logging.lastResort = None
    #     self.enableConsoleHandler()
    #     self.logger.addHandler(self.handlers.fileRotateHandler)

    def test(self):

        logger = self.get("loggerTest")

        self.enableConsoleMemHandler()

        logger.info("a test")

        self.enableMemHandler()

        def perftest(logger):
            print("start perftest logger")
            start = time.time()
            nr = 30000
            for i in range(nr):
                logger.info("this is an info message")
                # self.getActionObjFromMethod(test)
            stop = time.time()
            print("nr of logs per sec:%s" % int(nr / (stop - start)))

        perftest(logger)

        # FOLLOWING PROVES THAT THE LOOKING FOR FILE & PATH INFO IS THE SLOWING DOWN FACTOR
        # j.tools.performancetrace.profile("perftest(logger)", globals=locals())  # {"perftest": perftest}