import numpy as np
import random as rnd
import matplotlib.pyplot as plt
import math

xx=np.arange(0,100)
for i in range(99):
    xx=np.append(xx,np.arange(0,100))
yy=np.repeat(np.arange(0,100),100)

food=[]
for i in range(50):
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
food=np.append(food,np.zeros(1000))
food=food.astype(int)

males=[[],[]]
for i in range(9998):
    males.append([])
females=[[],[]]
for i in range(9998):
    females.append([])

grid=np.zeros((10000,4))
grid[0:10000,0]=np.ravel(xx)
grid[0:10000,1]=np.ravel(yy)
grid[0:10000,2]=np.ravel(food)

def grow_food():
    for i in range(0,10000):
        N=np.round((100*grid[i,2]*math.exp(0.8))/(100+(grid[i,2]*math.expm1(0.8))))
        grid[i,2]=N

def decay_pheromones():
    for i in range(0,10000):
        A=grid[i,3]*math.exp(-1)
        grid[i,3]=A

def list_sperm():
    for i in range(0,10000):
        if len(males[i])>0 and len(females[i])>0:
            for j in females[i]:
                for k in males[i]:
                    P1.pop[j].sperm.append(P1.pop[k].dauer_1)
                    P1.pop[j].sperm.append(P1.pop[k].dauer_2)

class Worm:
    species="C. nigoni"
    def __init__(self,name):
        self.name=name
        self.gender=rnd.choice(["hermaphrodite","male","female"])
        self.age=0
        self.stage="egg"
        self.location=[rnd.choice(xx),rnd.choice(yy)]
        self.energy=10
        self.L1_time=0
        self.L2d_time=0
        self.dauer_1=rnd.choice(np.arange(15,25))
        self.dauer_2=rnd.choice(np.arange(15,25))
        self.sperm=[]
    @property
    def position(self):
        return((100*self.location[1])+self.location[0])
    @property
    def dauer(self):
        return((self.dauer_1+self.dauer_2)/2)
    def move(self):
        if self.energy>0 or self.stage=="dauer":
            if self.stage!="dauer":
                self.energy=self.energy-0.5
            neighbors=[[(self.location[0]+1)%100,self.location[1]],[(self.location[0]-1)%100,self.location[1]],[self.location[0],(self.location[1]+1)%100],[self.location[0],(self.location[1]-1)%100]]
            list=[]
            for i in range(4):
                list.append(grid[100*neighbors[i][1]+neighbors[i][0],3])
            Sum=(1/(1+list[0]))+(1/(1+list[1]))+(1/(1+list[2]))+(1/(1+list[3]))
            P_0=(1/(1+list[0]))/Sum
            P_1=(1/(1+list[1]))/Sum
            P_2=(1/(1+list[2]))/Sum
            P_3=(1/(1+list[3]))/Sum
            if self.stage=="adult" and self.gender=="male":
                males[self.position].remove(self.name)
            if self.stage=="adult" and self.gender=="female":
                females[self.position].remove(self.name)
            index=np.random.choice([0,1,2,3],p=[P_0,P_1,P_2,P_3])
            self.location=neighbors[index]
            grid[self.position,3]=grid[self.position,3]+1
            if self.stage=="adult" and self.gender=="male":
                males[self.position].append(self.name)
            if self.stage=="adult" and self.gender=="female":
                females[self.position].append(self.name)
            print("The location of",self.name,"is",self.location)
            print("The amount of food is",grid[self.position,2].astype(int))
            print("The amount of pheromones is",grid[self.position,3])
        else:
            if self.stage=="adult" and self.gender=="male":
                males[self.position].remove(self.name)
            if self.stage=="adult" and self.gender=="female":
                females[self.position].remove(self.name)
            print("Worm",self.name,"has died of starvation")
            P1.pop[self.name]="dead starving worm"
    def eat(self):
        print("The location of",self.name,"is the same")
        if self.stage=="L1":
            portion=1
        if self.stage=="L2" or self.stage=="L2d":
            portion=2
        if self.stage=="L3":
            portion=4
        if self.stage=="dauer":
            portion=min(4,grid[self.position,2])
        if self.stage=="L4":
            portion=8
        if self.stage=="adult":
            portion=16
        if self.stage=="old":
            portion=8
        number=0
        while number<portion:
            if grid[self.position,2]==0:
                break
            if self.age==400:
                break
            if self.stage!="dauer":
                grid[self.position,2]=grid[self.position,2]-1
                self.energy=self.energy+1
            self.age=self.age+1
            self.grow()
            number=number+1
        if self.age<400:
            grid[self.position,3]=grid[self.position,3]+1
            print("The amount of food is",grid[self.position,2].astype(int))
            print("The amount of pheromones is",grid[self.position,3])
    def grow(self):
        if self.stage!="dauer":
            self.energy=self.energy-0.5
        if self.age==10:
            self.stage="L1"
            grid[self.position,3]=grid[self.position,3]+1
            print("Worm",self.name,"has reached L1")
        if self.age==20:
            prob=1/(1+math.exp(self.dauer-self.L1_time))
            self.stage=np.random.choice(["L2d","L2"],p=[prob,1-prob])
            print("Worm",self.name,"has reached",self.stage)
        if self.age==40 and self.stage=="L2":
            self.stage="L3"
            print("Worm",self.name,"has reached L3")
        if self.age==45 and self.stage=="L2d":
            prob=1/(1+math.exp(self.dauer-self.L2d_time))
            self.stage=np.random.choice(["dauer","L3"],p=[prob,1-prob])
            print("Worm",self.name,"has reached",self.stage)
            self.age=40
        if self.age==80:
            self.stage="L4"
            print("Worm",self.name,"has reached L4")
        if self.age==160:
            self.stage="adult"
            print("Worm",self.name,"has reached adulthood")
            if self.gender=="male":
                males[self.position].append(self.name)
            if self.gender=="female":
                females[self.position].append(self.name)
        if self.age==320:
            self.stage="old"
            print("Worm",self.name,"is old")
            if self.gender=="male":
                males[self.position].remove(self.name)
            if self.gender=="female":
                females[self.position].remove(self.name)
        if self.age==400:
            print("Worm",self.name,"has died of old age")
            P1.pop[self.name]="dead old worm"
    def reproduce(self):
        if self.stage=="adult" and self.gender=="hermaphrodite":
            count=0
            for i in range(len(P1.pop),len(P1.pop)+10):
                if self.energy>0:
                    self.energy=self.energy-0.5
                    P1.pop.append(Worm(i))
                    P1.pop[i].gender=np.random.choice(["hermaphrodite","male"],p=[0.99,0.01])
                    P1.pop[i].location=[self.location[0],self.location[1]]
                    P1.pop[i].dauer_1=np.random.choice([self.dauer_1,self.dauer_2,np.random.normal(self.dauer_1,0.5),np.random.normal(self.dauer_2,0.5)],p=[0.4995,0.4995,0.0005,0.0005])
                    P1.pop[i].dauer_2=np.random.choice([self.dauer_1,self.dauer_2,np.random.normal(self.dauer_1,0.5),np.random.normal(self.dauer_2,0.5)],p=[0.4995,0.4995,0.0005,0.0005])
                    count=count+1
            print(count,"eggs have been laid at",self.location)
        if self.stage=="adult" and self.gender=="female" and len(self.sperm)>0:
            count=0
            for i in range(len(P1.pop),len(P1.pop)+10):
                if self.energy>0:
                    self.energy=self.energy-0.5
                    P1.pop.append(Worm(i))
                    P1.pop[i].gender=np.random.choice(["male","female"])
                    P1.pop[i].location=[self.location[0],self.location[1]]
                    P1.pop[i].dauer_1=np.random.choice([self.dauer_1,self.dauer_2,np.random.normal(self.dauer_1,0.5),np.random.normal(self.dauer_2,0.5)],p=[0.4995,0.4995,0.0005,0.0005])
                    sperm=[]
                    prob=[]
                    for i in self.sperm:
                        sperm.append(i)
                        sperm.append(np.random.normal(i,0.5))
                        prob.append(0.999/len(self.sperm))
                        prob.append(0.001/len(self.sperm))
                    P1.pop[i].dauer_2=np.random.choice(sperm,p=prob)
                    count=count+1
            print(count,"eggs have been laid at",self.location)
    def decide(self):
        if self.stage=="L1":
            self.L1_time=self.L1_time+1
        if self.stage=="L2d":
            self.L2d_time=self.L2d_time+1
        if self.stage!="egg":
            food_weights=np.round(np.arange(1,-0.0001,-(1/100)),8)
            neighbors_weights=np.arange(0,1.0001,(1/100))
            F=grid[self.position,2]
            F=F.astype(int)
            P=grid[self.position,3]+grid[100*self.location[1]+((self.location[0]+1)%100),3]+grid[100*self.location[1]+((self.location[0]-1)%100),3]+grid[100*((self.location[1]+1)%100)+self.location[0],3]+grid[100*((self.location[1]-1)%100)+self.location[0],3]
            P=np.round(P)
            P=P.astype(int)
            if F==0:
                probability=1
            elif self.energy==0:
                probability=0
            elif P>99:
                probability=(food_weights[F]+1)/2
            else:
                probability=(food_weights[F]+neighbors_weights[P])/2
            if np.random.choice([True,False],p=[probability,1-probability])==True:
                self.move()
            else:
                self.eat()
            self.reproduce()
        if self.stage=="egg":
            print("Worm",self.name,"is an egg")
            self.age=self.age+1
            self.grow()
        if self.stage=="L2d" and (1/(1+math.exp(self.dauer-self.L2d_time)))>0.99:
            self.age=40
            self.stage="dauer"
            print("Worm",self.name,"has reached dauer")

class Population:
    def __init__(self,size):
        self.size=size
        self.pop=[]
    def populate(self):
        for i in range(0,self.size):
            self.pop.append(Worm(i))
            print("The location of",self.pop[i].name,"is",self.pop[i].location)
    def decide(self,worm_lower,worm_upper,decision):
        self.worm_lower=worm_lower
        self.worm_upper=worm_upper
        self.decision=decision
        number=0
        while number<self.decision:
            bounds=np.arange(self.worm_lower,self.worm_upper)
            rnd.shuffle(bounds)
            for i in bounds:
                try:
                    self.pop[i].decide()
                except AttributeError:
                    print("Worm",i,"is dead")
                    continue
            number=number+1
            list_sperm()
        grow_food()
        decay_pheromones()

def worm_map():
    x,y=np.mgrid[slice(0,100),slice(0,100)]
    z=grid[100*y+x,3]
    plt.pcolormesh(x,y,z,cmap='Blues')
    #plt.pcolormesh(x,y,z,cmap='Blues',vmin=0,vmax=100) to normalize
    plt.title('Pheromone Density')
    plt.xlabel('X Coordinate')
    plt.ylabel('Y Coordinate')
    plt.axis([0,100,0,100])
    plt.colorbar()

def food_map():
    x,y=np.mgrid[slice(0,100),slice(0,100)]
    z=grid[100*y+x,2]
    plt.pcolormesh(x,y,z,cmap='Blues')
    #plt.pcolormesh(x,y,z,cmap='Blues',vmin=0,vmax=100) to normalize
    plt.title('Food Density')
    plt.xlabel('X Coordinate')
    plt.ylabel('Y Coordinate')
    plt.axis([0,100,0,100])
    plt.colorbar()

def prob_move():
    x,y=np.mgrid[slice(0,101),slice(0,101)]
    z=np.zeros((101,101))
    for i in range(0,101):
        for j in range(0,101):
            food_weights=np.round(np.arange(1,-0.0001,-(1/100)),8)
            neighbors_weights=np.arange(0,1.0001,(1/100))
            if i==0:
                z[i,j]=1
            elif j>99:
                z[i,j]=(food_weights[i]+1)/2
            else:
                z[i,j]=(food_weights[i]+neighbors_weights[j])/2
    plt.pcolormesh(x,y,z,cmap='Blues')
    plt.title('Probability of Moving')
    plt.xlabel('Amount of Food')
    plt.ylabel('Amount of Pheromones')
    plt.axis([0,100,0,100])
    plt.colorbar(ticks=[np.arange(0,1.1,0.1)])

def how_many():
    egg,L1,L2,L2d,L3,dauer,L4,adult,old,dsw,dow=0,0,0,0,0,0,0,0,0,0,0
    for i in range(len(P1.pop)):
        if P1.pop[i]=="dead starving worm":
            dsw=dsw+1
        if P1.pop[i]=="dead old worm":
            dow=dow+1
        try:
            if P1.pop[i].stage=="egg":
                egg=egg+1
            if P1.pop[i].stage=="L1":
                L1=L1+1
            if P1.pop[i].stage=="L2":
                L2=L2+1
            if P1.pop[i].stage=="L2d":
                L2d=L2d+1
            if P1.pop[i].stage=="L3":
                L3=L3+1
            if P1.pop[i].stage=="dauer":
                dauer=dauer+1
            if P1.pop[i].stage=="L4":
                L4=L4+1
            if P1.pop[i].stage=="adult":
                adult=adult+1
            if P1.pop[i].stage=="old":
                old=old+1
        except AttributeError:
            continue
    print(egg,"egg")
    print(L1,"L1")
    print(L2,"L2")
    print(L2d,"L2d")
    print(L3,"L3")
    print(dauer,"dauer")
    print(L4,"L4")
    print(adult,"adult")
    print(old,"old")
    print(dsw,"dead starving worm")
    print(dow,"dead old worm")

def stats():
    gene_1=[]
    gene_2=[]
    for i in range(len(P1.pop)):
        try:
            gene_1.append(P1.pop[i].dauer_1)
            gene_2.append(P1.pop[i].dauer_2)
        except AttributeError:
            continue
    print("Gene 1 mean",np.mean(gene_1))
    print("Gene 2 mean",np.mean(gene_2))
    print("Gene 1 std",np.std(gene_1))
    print("Gene 2 std",np.std(gene_2))

P1=Population(1000)
P1.populate()


from django.urls import path
from . import views

urlpatterns = [
	path('',views.index,name='index'),
	path('list/',views.blog_list,name='blog_list'),
	path('item/<int:blog_id>',views.blog_item,name='blog_item'),
	path('post/',views.blog_post,name='blog_post'),
	path('post/submit/',views.blog_post_submit,name='blog_post_submit'),
	path('edit/<int:blog_id>',views.blog_edit,name='blog_edit'),
	path('update/<int:blog_id>',views.blog_update,name='blog_update'),
	path('delete/<int:blog_id>',views.blog_delete,name='blog_delete'),
]


import sys
from flask import Flask, jsonify, make_response, request, abort

app = Flask(__name__)

@app.after_request
def after_request(data):
	response = make_response(data)
	response.headers['Content-Type'] = 'application/json'
	response.headers['Access-Control-Allow-Origin'] = '*'
	response.headers['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept"
	response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE'
	return response

tasks = []

@app.errorhandler(400)
def bad_request(error):
    return make_response(jsonify( { 'error': 'Bad Request' } ), 400)

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify( { 'error': 'Not Found' } ), 404)

@app.route('/tasks', methods = ['POST'])
def create_task():
    if not request.json or not 'task' in request.json:
        abort(400)
    task = {
        'id': len(tasks),
        'task': request.json['task']
    }
    tasks.append(task)
    return jsonify( { 'task': task } ), 201

@app.route('/tasks', methods = ['GET'])
def get_tasks():
    return jsonify( { 'tasks': tasks } )

@app.route('/tasks/<int:task_id>', methods = ['GET'])
def get_task(task_id):
    task = filter(lambda t: t['id'] == task_id, tasks)
    if len(task) == 0:
        abort(404)
    return jsonify( { 'task': task[0] } )

@app.route('/tasks/<int:task_id>', methods = ['PUT'])
def update_task(task_id):
    task = filter(lambda t: t['id'] == task_id, tasks)
    if len(task) == 0 or not request.json:
        abort(404)
    if 'task' in request.json and type(request.json['task']) is not unicode:
        abort(400)
    task[0]['task'] = request.json.get('task', task[0]['task'])
    return jsonify( { 'task': task[0] } )

@app.route('/tasks/<int:task_id>', methods = ['DELETE'])
def delete_task(task_id):
    task = filter(lambda t: t['id'] == task_id, tasks)
    if len(task) == 0:
        abort(404)
    tasks.remove(task[0])
    return jsonify( { 'result': True } )

if __name__ == '__main__':
    app.run(debug = True)


# Bayesian networks with pomegranate

import numpy as np
from pomegranate import *

def montyhall():
    # The doors are named A, B and C
    # The door selected by the guest and the door hiding the prize are random
    # There are three nodes, each having as value one of the doors A, B or C
    # guest = n1: the guest's choice
    # prize = n2: the door hiding the prize
    # monty = n3: the door Monty picks depending on guest and prize
    # There are two edges: [n1, n3] and {n2, n3]

    guest = DiscreteDistribution({'A': 1 / 3, 'B': 1 / 3, 'C': 1 / 3})
    prize = DiscreteDistribution({'A': 1 / 3, 'B': 1 / 3, 'C': 1 / 3})
    monty = ConditionalProbabilityTable(
        [['A', 'A', 'A', 0.0],  # P(monty = A | guest = A, prize = A)
         ['A', 'A', 'B', 0.5],  # P(monty = B | guest = A, prize = A)
         ['A', 'A', 'C', 0.5],  # P(monty = C | guest = A, prize = A)
         ['A', 'B', 'A', 0.0],  # P(monty = A | guest = A, prize = B)
         ['A', 'B', 'B', 0.0],
         ['A', 'B', 'C', 1.0],
         ['A', 'C', 'A', 0.0],
         ['A', 'C', 'B', 1.0],
         ['A', 'C', 'C', 0.0],
         ['B', 'A', 'A', 0.0],
         ['B', 'A', 'B', 0.0],
         ['B', 'A', 'C', 1.0],
         ['B', 'B', 'A', 0.5],
         ['B', 'B', 'B', 0.0],
         ['B', 'B', 'C', 0.5],
         ['B', 'C', 'A', 1.0],
         ['B', 'C', 'B', 0.0],
         ['B', 'C', 'C', 0.0],
         ['C', 'A', 'A', 0.0],
         ['C', 'A', 'B', 1.0],
         ['C', 'A', 'C', 0.0],
         ['C', 'B', 'A', 1.0],
         ['C', 'B', 'B', 0.0],
         ['C', 'B', 'C', 0.0],
         ['C', 'C', 'A', 0.5],
         ['C', 'C', 'B', 0.5],
         ['C', 'C', 'C', 0.0]], [guest, prize])

    n1 = Node(guest, name="guest")
    n2 = Node(prize, name="prize")
    n3 = Node(monty, name="monty")

    network = BayesianNetwork("Monty Hall Problem")
    network.add_nodes(n1, n2, n3)
    network.add_edge(n1, n3)
    network.add_edge(n2, n3)

    network.bake()

    # print(network.dense_transition_matrix())

    print(network.probability([['A', 'A', 'A'],
                               ['A', 'A', 'B'],
                               ['C', 'C', 'B'],
                               ['A', 'B', 'C']]))

    print(network.predict_proba([None, None, None]))
    print(network.predict_proba(['A', None, None]))
    print(network.predict_proba({'guest': 'A', 'monty': 'C'}))

def morse():
    # dots and dashes are being sent and received

    sender = DiscreteDistribution({"dot": 0.7, "dash": 0.3})
    receiver = ConditionalProbabilityTable(
        [["dot", "dot", 0.99],
         ["dot", "dash", 0.01],
         ["dash", "dot", 0.02],
         ["dash", "dash", 0.98]], [sender])

    n1 = Node(sender, name="sender")
    n2 = Node(receiver, name="receiver")

    network = BayesianNetwork("Morse")
    network.add_nodes(n1, n2)
    network.add_edge(n1, n2)

    network.bake()

    # print(network.probability([['dot', 'dot'],                  # P(dot, dot): dot sent, dot received
    #                             ['dot', 'dash'],                # P(dash, dot): dash sent, dot received
    #                             ['dash', 'dot'],                # P(dot, dash): dash sent, dot received
    #                             ['dash', 'dash']]))             # P(dash, dash): dash sent, dash received

    # print(network.predict_proba([None, None]))                  # returns marginal distribution of dot, dash
    # print(network.marginal())                                     # returns marginal distribution of dot, dash
    #
    # print(network.predict_proba(['dot', None]))                 # forward probabilities given sender = dot
    print(network.predict_proba({'sender': 'dot'}))  # forward probabilities given sender = dot
    # print(network.predict_proba([None, 'dot']))                 # backward probabilities given receiver = dot
    # print(network.predict_proba({'receiver': 'dot'}))           # backward probabilities given receiver = dot
    # print(network.predict_proba(['dot', 'dot']))                # returns just ['dot', 'dot']

def samples():
    numpy.random.seed(111)

    X = numpy.random.randint(2, size=(15, 15))
    X[:, 5] = X[:, 4] = X[:, 3]
    X[:, 11] = X[:, 12] = X[:, 13]

    network = BayesianNetwork.from_samples(X)

def fitting():
    samples1 = np.array([['x', 'x'], ['x', 'x'], ['x', 'y'], ['x', 'y'], ['y', 'y']])
    samples2 = np.array([['x', 'x'], ['x', 'y']])

    samples3 = np.array([['x', 'x'], ['x', 'x'], ['x', 'y'], ['x', 'y'],
                         ['y', 'x'], ['y', 'x'], ['y', 'y'], ['y', 'y']])

    a = DiscreteDistribution({'x': 0.5, 'y': 0.5})
    b = ConditionalProbabilityTable(
        [['x', 'x', 0.5],
         ['x', 'y', 0.5],
         ['y', 'x', 0.5],
         ['y', 'y', 0.5]], [a])

    n1 = Node(a, name="a")
    n2 = Node(b, name="b")

    network = BayesianNetwork('Test')
    network.add_nodes(n1, n2)
    network.add_edge(n1, n2)

    network.bake()
    network.summarize(samples1)
    # network.from_summaries()
    print(network.probability([['x', 'x'],
                               ['x', 'y'],
                               ['y', 'x'],
                               ['y', 'y']]))

    network.summarize(samples2)
    network.from_summaries()
    print(network.probability([['x', 'x'],
                               ['x', 'y'],
                               ['y', 'x'],
                               ['y', 'y']]))
    # network.summarize(samples3)
    # print(network.probability([['x', 'x'],
    #                            ['x', 'y'],
    #                            ['y', 'x'],
    #                            ['y', 'y']]))

def fitting1():
    samples1 = np.array([[0, 0], [0, 0], [0, 1], [0, 1], [1, 1]])
    samples2 = np.array([[0, 0], [1, 1]])

    # samples3 = np.array([[0, 0], [0, 0], [0, 1], [0, 1],
    #                      [1, 0], [1, 0], [1, 1], [1, 1]])
    #
    # a = DiscreteDistribution({0: 0.9, 1: 0.1})
    # # b = DiscreteDistribution({0: 0.5, 1: 0.5})
    #
    # b = ConditionalProbabilityTable(
    #     [[0, 0, 0.5],
    #      [0, 1, 0.5],
    #      [1, 0, 0.5],
    #      [1, 1, 0.5]], [a])
    #
    # n1 = Node(a, name="a")
    # n2 = Node(b, name="b")

    network = BayesianNetwork.from_structure(structure=((), (0,)))
    # network.add_nodes(n1, n2)
    # network.add_edge(n1, n2)

    # network.bake()
    network.fit(samples2)

    print(network.probability([[0, 0],
                               [0, 1],
                               [1, 0],
                               [1, 1]]))

    print(network.marginal())

if __name__ == '__main__':
    # montyhall()
    # morse()
    # mini()
    # fitting1()
    # from_sample_struct()
    from_sample()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import math
import torch.utils.model_zoo as model_zoo

from .blocks import *

BN_MOMENTUM = 0.1

# __all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
#         'resnet152']

model_urls = {
    'resnet18': 'https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth',
    'resnet34': 'https://s3.amazonaws.com/pytorch/models/resnet34-333f7ec4.pth',
    'resnet50': 'https://s3.amazonaws.com/pytorch/models/resnet50-19c8e357.pth',
    'resnet101': 'https://s3.amazonaws.com/pytorch/models/resnet101-5d3b4d8f.pth',
    'resnet152': 'https://s3.amazonaws.com/pytorch/models/resnet152-b121ed2d.pth',
}

class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes=1000):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                    bias=False)
        self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, num_blocks, stride=1):
        layers = []
        layers.append(block(self.inplanes, planes, stride))
        self.inplanes = planes * block.expansion
        for i in range(1, num_blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

def resnet18(pretrained=False):
    """Constructs a ResNet-18 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(BasicBlock, [2, 2, 2, 2])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
    return model

def resnet34(pretrained=False):
    """Constructs a ResNet-34 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(BasicBlock, [3, 4, 6, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet34']))
    return model

def resnet50(pretrained=False):
    """Constructs a ResNet-50 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 4, 6, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet50']))
    return model

def resnet101(pretrained=False):
    """Constructs a ResNet-101 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 4, 23, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet101']))
    return model

def resnet152(pretrained=False):
    """Constructs a ResNet-152 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 8, 36, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet152']))
    return model

class deconv_resnet(nn.Module):
    def __init__(self, num_layers=50, num_classes=17, pretrained=False):
        super(deconv_resnet, self).__init__()
        num_feats = 256
        bias = False

        if num_layers == 50:
            self.resnet = resnet50()
            # self.resnet = resnet50(pretrained)
            model_path = 'data/pretrained/resnet50.pth'
        elif num_layers == 101:
            self.resnet = resnet101()
            # self.resnet = resnet101(pretrained)
            model_path = 'data/pretrained/resnet101.pth'
        elif num_layers == 152:
            self.resnet = resnet152()
            # self.resnet = resnet152(pretrained)
            model_path = 'data/pretrained/resnet152.pth'

        if pretrained:
            print("Loading pretrained weights from %s" %(model_path))
            state_dict = torch.load(model_path)
            self.resnet.load_state_dict({k:v for k,v in state_dict.items() if k in self.resnet.state_dict()})

        self.backbone = nn.Sequential(self.resnet.conv1, self.resnet.bn1, self.resnet.relu, self.resnet.maxpool,
                                      self.resnet.layer1, self.resnet.layer2, self.resnet.layer3, self.resnet.layer4)

        self.deconv = nn.Sequential(
                    nn.ConvTranspose2d(2048, num_feats, kernel_size=4, stride=2, padding=1, bias = bias),
                    nn.BatchNorm2d(num_feats, momentum=BN_MOMENTUM),
                    nn.ReLU(inplace=True),
                    nn.ConvTranspose2d(num_feats, num_feats, kernel_size=4, stride=2, padding=1, bias=bias),
                    nn.BatchNorm2d(num_feats, momentum=BN_MOMENTUM),
                    nn.ReLU(inplace=True),
                    nn.ConvTranspose2d(num_feats, num_feats, kernel_size=4, stride=2, padding=1, bias=bias),
                    nn.BatchNorm2d(num_feats, momentum=BN_MOMENTUM),
                    nn.ReLU(inplace=True) )

        self.heatmap = nn.Conv2d(num_feats, num_classes, kernel_size=1)

        for m in self.deconv.modules():
            if isinstance(m, nn.ConvTranspose2d):
                m.weight.data.normal_(0, 0.001)
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # m.weight.data.normal_(0, math.sqrt(2. / n))
                # bound = math.sqrt(6. / n)
                # m.weight.data.uniform_(-bound, bound)
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

        for m in self.heatmap.modules():
            if isinstance(m, nn.Conv2d):
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # m.weight.data.normal_(0, math.sqrt(2. / n))
                m.weight.data.normal_(0, 0.001)
                m.bias.data.zero_()

        def set_bn_fix(m):
            classname = m.__class__.__name__
            if classname.find('BatchNorm') != -1:
                for p in m.parameters(): p.requires_grad=False

        # self.resnet.apply(set_bn_fix)

    def forward(self, x):
        x = self.backbone(x)
        x = self.deconv(x)
        out = self.heatmap(x)

        return out

    # def train(self, mode=True):
    #     # Override train so that the training mode is set as we want
    #     nn.Module.train(self, mode)
    #     if mode:
    #         # Set fixed blocks to be in eval mode (not really doing anything)
    #         self.resnet.eval()
    #         if self.FIXED_BLOCKS <= 3:
    #             self.resnet.layer4.train()
    #         if self.FIXED_BLOCKS <= 2:
    #             self.resnet.layer3.train()
    #         if self.FIXED_BLOCKS <= 1:
    #             self.resnet.layer2.train()
    #         if self.FIXED_BLOCKS == 0:
    #             self.resnet.layer1.train()

    #     # Set batchnorm always in eval mode during training
    #     def set_bn_eval(m):
    #         classname = m.__class__.__name__
    #         if classname.find('BatchNorm') != -1:
    #         m.eval()

    #     self.resnet.apply(set_bn_eval)

    def load_pretrained_resnet(self, state_dict):
        self.resnet.load_state_dict({k: state_dict[k] for k in list(self.resnet.state_dict())})


from config import Configuration
from embedder import train_embedder
import os
import pandas as pd

data_name = 'CarEvaluation'
print(data_name)

# data = pd.read_csv('data/{}.csv'.format(data_name)).drop('class',axis=1)
data = pd.read_csv('data/{}.csv'.format(data_name))

embedding_size = 4
corruption_ratio_train = 0.0
corruption_ratio_validation = 0.05
config_data = Configuration(train_batch_size=128, val_batch_size= int(len(data) * 0.2),
                            embedding_size=embedding_size, max_iteration=20000,
                            print_loss_every=100,
                            LOG_DIR = './{}/embedding_{}/corruption_{}/validation_{}/class_var/log/'.format(
                                data_name, embedding_size, corruption_ratio_train,
                                corruption_ratio_validation),
                            corruption_ratio_train=corruption_ratio_train,
                            corruption_ratio_validation = corruption_ratio_validation,
                            model_save_filename = 'model.ckpt',
                            metadata_filename = 'metadata.tsv',
                            )

if not os.path.exists(config_data.LOG_DIR):
    os.makedirs(config_data.LOG_DIR)

with open(os.path.join(config_data.LOG_DIR, 'config.txt'), 'w') as f:
    f.write(repr(config_data))

for col in data.columns:
    if data[col].dtype == int or data[col].dtype == float:
        data[col] = data[col].astype('str')

train_embedder(data, config_data)


import tensorflow.keras as kr
import numpy as np

#Create arrays
nn = [2, 4, 6, 8, 12, 10, 8, 2]
labels = []
words = []
docs_x = []
docs_y = []

INIT_LR = 1e-3
epochs = 6
batch_size = 64

training_data = np.array(np.random.random((100, 32)), dtype=np.float32)
target_data = np.random.randint(0, 2, 100, dtype=np.int32)
target_data = np.eye(2)[target_data]

model = kr.Sequential()

model.add(kr.layers.Dense(nn[1], activation='relu', input_shape=(32,)))
model.add(kr.layers.Dense(nn[2], activation='relu'))
model.add(kr.layers.Dense(nn[3], activation='relu'))
model.add(kr.layers.Dense(nn[4], activation='relu'))
model.add(kr.layers.Dense(nn[5], activation='relu'))
model.add(kr.layers.Dense(nn[6], activation='relu'))
model.add(kr.layers.Dense(nn[7], activation='sigmoid'))

print(model.summary())
model.compile(loss=kr.losses.categorical_crossentropy, optimizer=kr.optimizers.Adagrad(lr=INIT_LR, decay=INIT_LR / 100),metrics=['accuracy'])

#Training
model.fit(training_data , target_data , epochs=1)


class Node:
    def __init__(self, item, next):
        self.item = item
        self.next = next

class LinkedList:
    def __init__(self):
        self.head = None

    def add(self, item):
        self.head = Node(item, self.head)

    def remove(self):
        if self.is_empty():
            return None
        else:
            item = self.head.item
            self.head = self.head.next    # remove the item by moving the head pointer
            return item

    def is_empty(self):
        return self.head == None

    def last_element(self):
        ptr = self.head
        while ptr.next != None:
            prev = ptr
            ptr = ptr.next
        prev.next = None

    def remove_negs(self):
        ptr = self.head
        x = 0
        while ptr.next != None:
            if ptr == x:
                self.remove(ptr.next)
                ptr = ptr.next
            else:
                ptr = ptr.next

    def __str__(self):
        tmp_str = ""
        ptr = self.head
        while ptr != None:
            tmp_str += ptr.item + " "
            ptr = ptr.next

        return tmp_str

import sys
from lastyear_examq1 import LinkedList

def main():
    # Read data from input
    ll = LinkedList()
    ll.add(1)
    ll.add(2)
    ll.add(-3)
    ll.add(4)
    print(ll)
    #print(str(ll.remove()) + '\n')
    ll.remove_negs()
    print(ll)

if __name__ == "__main__":
    main()


from heapq import *
from time import gmtime, strftime, sleep

from IO import *

def cache_to_endpoint(n_cache, endpoints):
    res = np.zeros(shape=(n_cache, len(endpoints)), dtype=np.bool)

    for ep in endpoints:
        for c in ep.con:
            res[c[0]][ep.id] = True

    return res

def vid_to_endpoint(n_vid, n_end, requests):
    res = np.zeros(shape=(n_vid, n_end), dtype=np.uint32)

    for r in requests:
        res[r.vid][r.eid] = r.n

    return res

def solution(graph):
    caches_count = len(graph['caches'])
    # solutions
    videos_on_cache = [[] for _ in range(caches_count)]
    scores = {}

    ScoreKey = namedtuple('ScoreKey', ['video', 'cache'])

    for cid in tqdm(range(caches_count), desc="Computing Scores"):
        scores_for_c = {}
        cache = graph['caches'][cid]
        # if there are requests, compute score
        for eid in cache['endpoints']:
            ep = graph['endpoints'][eid]
            lat_diff = ep['latency']
            for con in ep['connections']:
                if graph['cache_mapping'][con[0]] == cid:
                    lat_diff -= con[1]
                    break

            for rid in ep['requests']:
                req = graph['requests'][rid]
                key = ScoreKey(req.vid, cid)
                score = 0.0
                if key in scores_for_c:
                    score = scores_for_c[key]
                score -= req.n * lat_diff / graph['videos'][req.vid]['size']
                scores_for_c[key] = score

                del req
                del key
                del score

            del ep

        scores.update(scores_for_c)

        del scores_for_c
        del cache

    pq = []
    for k in tqdm(scores, desc="Filling Priority Queue"):
        heappush(pq, (scores[k], k.video, k.cache))

    # now we have the heap with the scores
    # print("Size of the heap", len(pq), "/", str(len(graph['videos']) * len(graph['caches'])))

    # update from here on
    pbar = tqdm(total=len(pq), desc="Progressing Priority Queue")
    while pq:
        (s, v, c) = heappop(pq)
        key = ScoreKey(v, c)
        # check if score has changed
        # push the tuple if score has changed
        if scores[key] > s:
            heappush(pq, (scores[key], v, c))
            continue

        pbar.update(1)
        cache_size = np.sum([graph['videos'][vids]['size'] for vids in videos_on_cache[c]])
        if cache_size + graph['videos'][v]['size'] > graph['caches'][c]['size']:
            # continue if cache is full
            continue

        if v in videos_on_cache[c]:
            print("ERROR", "Duplicate video index", v, "in", c)
            break

        videos_on_cache[c].append(v)

        # now update neighboring cache which need to hold the same video
        cache = graph['caches'][c]
        for eid in cache['endpoints']:
            ep = graph['endpoints'][eid]
            for ncon in ep['connections']:
                nc = graph['cache_mapping'][ncon[0]]
                if nc == c:
                    continue
                other = ScoreKey(v, nc)
                if other in scores:

                    lat_diff = ep['latency']
                    for con in ep['connections']:
                        if graph['cache_mapping'][con[0]] == nc:
                            lat_diff -= con[1]
                            break

                    n_requests = 0
                    for rid in graph['videos'][v]['requests']:
                        req = graph['requests'][rid]
                        if req.eid == eid:
                            n_requests = req.n
                            break

                    score = scores[other]
                    score += n_requests * lat_diff / graph['videos'][v]['size']
                    scores[other] = score

    return videos_on_cache

def solution_old(n_vid, n_end, n_req, n_cache, s_cache, s_videos, endpoints, requests):
    # solutions
    cache = np.zeros(n_cache)
    videos_on_cache = [[] for _ in range(n_cache)]
    # compute scores
    scores = np.zeros(shape=(n_vid, n_cache), dtype=np.double)
    in_q = np.zeros(shape=(n_vid, n_cache), dtype=np.double)

    pq = []

    c2e = cache_to_endpoint(n_cache, endpoints)
    v2e = vid_to_endpoint(n_vid, n_end, requests)

    for req in tqdm(requests, desc="Computing scores for Video/Cache Pairs"):
        ep = endpoints[req.eid]
        for c in ep.con:
            d_latency = ep.lat - c[1]
            score = req.n * d_latency * (1.0 / s_videos[req.vid])
            scores[req.vid][c[0]] -= score

    for req in tqdm(requests, desc="Pushing Video/Cache Pairs into Priority Queue"):
        ep = endpoints[req.eid]
        for c in ep.con:
            if not in_q[req.vid][c[0]]:
                in_q[req.vid][c[0]] = True
                heappush(pq, (scores[req.vid][c[0]], req.vid, c[0]))

    tqdm.write(str(100 * (len(pq)) / (n_vid * n_cache)) + "% in Queue.")
    sleep(0.1)
    pbar = tqdm(total=len(pq), desc="Progressing Priority Queue")
    # update from here on
    while pq:
        pbar.update(1)
        (s, v, c) = heappop(pq)

        # check if score has changed
        if any(v in cache_ for cache_ in videos_on_cache):
            if scores[v][c] > s:
                heappush(pq, (scores[v][c], v, c))

        if cache[c] + s_videos[v] <= s_cache:
            videos_on_cache[c].append(v)
            cache[c] += s_videos[v]

            if not True:
                # update scores for connected caches / videos
                for req in requests:
                    if req.vid == v:
                        ep = endpoints[req.eid]
                        for ca in ep.con:
                            d_latency = ep.lat - ca[1]
                            score = v2e[v][req.eid] * d_latency * (1.0 / s_videos[v])
                            # is this the right one?
                            scores[v][ca[0]] = 0  # score
            else:
                # update scores for connected caches / videos
                for eid in range(n_end):
                    # endpoint connects to cache
                    if c2e[c][eid]:
                        # if video is requested by the endpoint
                        if v2e[v][eid]:
                            ep = endpoints[eid]
                            for ec in ep.con:
                                d_latency = ep.lat - ec[1]
                                score = v2e[v][eid] * d_latency * (1.0 / s_videos[v])
                                scores[v][ec[0]] = 0  # score

        if len(pq) % 1000 == 0:
            write_solution(strftime("%Y%m%d-%H%M%S", gmtime()), cache, videos_on_cache)

    return cache, videos_on_cache


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'index', views.index, name='index'),
    url(r'create', views.create_role, name='create'),
    url(r'update', views.update_role, name='update'),
    url(r'delete', views.delete_role, name='delete'),
    url(r'assign', views.assign_role, name='assign'),
    url(r'deassign', views.deassign_role, name='deassign'),
]


#!/user/bin/python3.4
import os
import glob
import subprocess
import re

grader_log = open("grader-script-log.txt", "w")
grader_log.write("Grading script log:\n")

def log(msg):
    grader_log.write(msg+"\n")

def record_score_and_exit(score):
    score_file = open("score.dat", "w")
    score_file.write(str(score) + "\n")
    score_file.close()
    grader_log.close()
    exit()

def unpack_and_compile(package_names, support_package_path=""):
    # Unpack JAR file
    jar_files = glob.glob("*.jar") + glob.glob("*.JAR")

    if len(jar_files) == 0:
        log("Couldn't find jar file.")
        log("Looked for files ending .jar and .JAR")
        log("If your submission file was not a jar file or did not end in .jar or .JAR, please correct and resubmit.")
        record_score_and_exit(0)

    if subprocess.call(["jar", "xvf", jar_files[0]]) != 0:
        log("Unpacking jar file failed. Grade script could not continue.")
        record_score_and_exit(0)

    log("Unpacked jar file: " + jar_files[0])

    if not os.path.exists("bin"):
        os.mkdir("bin")
    else:
        subprocess.call("rm -rf bin/*", shell=True)

    bin_dir = os.path.join(os.getcwd(), "bin")

    for pkg in package_names:
        log("\nLooking for package: " + pkg)
        find_result = subprocess.check_output(["find", ".", "-type", "d", "-name", pkg], universal_newlines=True).strip()

        if find_result != "":
            pkg_dir = find_result.split('\n')[0]
            log("Found " + pkg + " directory, compiling...")
            topdir = os.getcwd()
            os.chdir(pkg_dir)
            cp_dir = bin_dir
            if support_package_path != "":
                cp_dir = cp_dir+":"+support_package_path

            cmd = "/usr/java/bin/javac -d " + bin_dir + " -cp " + cp_dir + " *.java"
            try:
                output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
                log("...successful.")
            except subprocess.CalledProcessError as exc:
                log("...failed!")
                log("There was a problem in compiling the code in the package " + pkg + ".")
                log("One possible cause is that you did not include the .java source files in your JAR file.")
                log("Another possibility is that your code contains syntax errors.")
                log("The grading script will still attempt to run but may not work.")
                log("Plese resubmit when fixed.")
                log("Here is the output from the compiler:")
                log(exc.output.decode("utf-8"))

            os.chdir(topdir)
        else:
            log("Did not find package "+ pkg)
            log("Double check your code for the correct package name and resubmit.")
            log("The grading script will still attempt to run but may not work.")

def run_with_io(program_name, input_str, out_re, out_exemplar, class_path=None, args=None, max_time=10):
    log("\nRunning " + program_name)
    log("Input: ")
    log(input_str)
    cp = "bin"
    if class_path != None:
        cp = cp + ":" + class_path

    cmd = ["/usr/java/bin/java", "-cp", cp, program_name]
    if args != None:
        cmd = cmd + args

    op = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
    try:
        output, err = op.communicate(input=input_str, timeout=max_time)

    except subprocess.TimeoutExpired:
        log("Timeout expired. Your program seems to be in an endless loop.")
        return False

    log("\nOutput:")
    log(output)

    if out_re != None:
        match = re.match(out_re, output, flags=re.M)
        if match == None:
            log("\nOutput did not match expectations.")
            if out_exemplar != None:
                log("Expected output:")
                log(out_exemplar)
            return False
        else:
            log("Output matches expectations.")

    return True

def run_tester(class_path, tester):
    log("\nRunning " + tester)
    cmd = ["/usr/java/bin/java", "-cp", "bin:/var/www/grade/comp401sp16/junit/*:"+class_path, tester]
    try:
        output = subprocess.check_output(cmd, universal_newlines = True, timeout = 100, stderr=subprocess.STDOUT)
    except subprocess.TimeoutExpired:
        log("Timeout expired. Your program sees to be in an endless loop.")
        return 0
    except subprocess.CalledProcessError as exc:
        log("Your code has crashed the tester. This usually happens if your code has caused a runtime exception such as a NullPointerException.")
        log("Output from test:")
        log(exc.output)
        return 0

    log(output)

    return float(output.split('\n')[-2].split(':')[-1].strip())


import time
link = "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/"

class TestCatalogePage():
    def test_page_contains_button_to_add_a_product_to_the_basket(self, browser):
        browser.get(link)
        time.sleep(5)
        assert browser.find_element_by_css_selector("button.btn-add-to-basket"), "The page is missing a button to add" \
                                                                                " a product to the basket"


import sys
import traceback
from gaeparallelize.models import LogRecord

class Logger(object):
    Location = LogRecord.Location

    def __init__(self, job_key, location):
        if location not in LogRecord.Location.values:
            raise ValueError('location must be one of %r'
                             % LogRecord.Location.values)
        self.job_key = job_key
        self.location = location

    def _log(self, level, message=None, error=None):
        if not message and not error:
            raise ValueError('both message and error are not present')
        if not message:
            message = str(error)
        stack_trace_lines = None
        if error:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            stack_trace_lines = traceback.format_tb(exc_traceback)
        LogRecord(
            job_key=self.job_key,
            location=self.location,
            level=level,
            message=message,
            stack_trace_lines=stack_trace_lines
        ).put()

    def info(self, message):
        self._log(LogRecord.Level.INFO, message)

    def warning(self, message=None, error=None):
        self._log(LogRecord.Level.WARNING, message, error)

    def error(self, message=None, error=None):
        self._log(LogRecord.Level.ERROR, message, error)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 10 10:30:09 2019

@author: koch
"""

# Import libraries

# Misc
from pathlib import Path
from os import listdir
from os import system
from copy import deepcopy
from sys import argv

# Math
import numpy as np
import scipy.io
import scipy.stats
from random import sample
from collections import Counter

# Image processing
import nibabel
from nilearn import signal
from nilearn import image
import pandas as pd

# Machine learning
import sklearn.model_selection
import sklearn.svm

# Plotting
import matplotlib.pyplot as plt
from matplotlib import gridspec

''' # Define functions used in this script '''
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# 1. Create_raw_matrix
# Function to create matrix for classifier test on raw data (each TR one row)
# - Define path to raw data
# - Load smoothed or unsmoothed raw_data
# - Transform into test matrix (2D array: 1st D voxel array, 2nd D TR)
# - Return test matrix of raw data for each run and mask in list of lists
#       - 1st list: Run 1, Run 2
#       - 2nd lst: Mask 1, ..., Mask n
#       - Example: output[0][4] = raw_mat of Run 1, 5th Mask
#       - Example: output[1][0] = raw_mat of Run 2, 1st Mask

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. masks: 3D array of mask values in 1s and 0s, 4D array in case of
# multiple masks
# 4. preproc: Logical to pick preprocessed or unprocessedraw data
#       - preproc already corrected, smoothed, and z-scored the data and
#       eliminated TRs which are not in the behavioral data
#       - 0: Use unprocessed data
#       - 1: Use preprocessed data
# 5. smooth: In case of unprocessed data. Boolean to determine if data should
# be smoothed
#       - 0: Data will not be smoothed
#       - 1: Data will be smoothed
# 6. smooth_kernel: Number to determine kernel size of smoothing (in case
# data will be smoothed)
# 7. concat: Boolean to determine if separated runs should be returned
# concatenated or as separated runs (in list format)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Create_raw_matrix(bids_dir, sub_id, masks, preproc, smooth, smooth_kernel,
                      concat):

    # Pose input of masks in appropriate format in case only one mask is
    # given
    if len(masks.shape) == 3:
        # Add 4th dimension and fill first 3 with provided mask
        pattern = np.zeros([masks.shape[0], masks.shape[1], masks.shape[2], 1])
        pattern[:,:,:,0] = masks
        # Make new 4D array to masks
        masks = pattern

    # Provide path to raw data dependent on taking preprocessed or unprocessed
    # data
    if bool(preproc):
        raw_dir_r1 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/classification_raw/zscsar' + sub_id +
                      '_task-feedback1_bold.nii')
        raw_dir_r2 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/classification_raw/zscsar' + sub_id +
                      '_task-feedback2_bold.nii')
    else:
        raw_dir_r1 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/smooth/sar' + sub_id +
                      '_task-feedback1_bold.nii')
        raw_dir_r2 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/smooth/sar' + sub_id +
                      '_task-feedback2_bold.nii')

    # Load selected raw data into matrix
    nii_raw_r1 = nibabel.load(raw_dir_r1)
    nii_raw_r2 = nibabel.load(raw_dir_r2)

    # Smooth data with provided kernel size in case unprocessed data is loaded
    if bool(smooth):
        nii_raw_r1 = image.smooth_img(nii_raw_r1, smooth_kernel)
        nii_raw_r2 = image.smooth_img(nii_raw_r2, smooth_kernel)

    # Cast nii data into matrix
    volume_raw_r1 = nii_raw_r1.get_data()
    volume_raw_r2 = nii_raw_r2.get_data()

    # Ger number of TRs in each run
    n_tr_r1 = volume_raw_r1.shape[3]
    n_tr_r2 = volume_raw_r2.shape[3]

    # Allocate list to store raw matrices for different masks
    masked_raw_mat_r1 = [0] * masks.shape[3]
    masked_raw_mat_r2 = deepcopy(masked_raw_mat_r1)
    # Allocate list for concatenated runs
    masked_raw_mat_concat = deepcopy(masked_raw_mat_r1)

    # Create different raw matrix for each mask
    for mask_count in np.arange(masks.shape[3]):

        # Get current mask
        current_mask = masks[:,:,:,mask_count]

        # Allocate 2D array: 1st D holding all non-zero masked voxels, 2nd D
        # holding different TRs
        current_raw_mat_r1 = np.zeros([np.flatnonzero(current_mask).size,
                                       n_tr_r1])
        current_raw_mat_r2 = np.zeros([np.flatnonzero(current_mask).size,
                                       n_tr_r2])

        # Fill allocated raw matrices with masked TRs (Separated for runs
        # since number of TRs differs)
        # Run 1
        for tr_count in np.arange(volume_raw_r1.shape[3]):

            # Isolate current TR
            current_tr = volume_raw_r1[:,:,:,tr_count]

            # Fill raw matrix with masked values for current TR
            current_raw_mat_r1[:,tr_count] = (
                    current_tr[np.nonzero(current_mask)]
                    )

        # Run 2
        for tr_count in np.arange(volume_raw_r2.shape[3]):
            current_tr = volume_raw_r2[:,:,:,tr_count]
            current_raw_mat_r2[:,tr_count] = (
                    current_tr[np.nonzero(current_mask)]
                    )

        # Transpose to make different TRs the rows of the matrix
        current_raw_mat_r1 = np.transpose(current_raw_mat_r1, (1,0))
        current_raw_mat_r2 = np.transpose(current_raw_mat_r2, (1,0))

        # Fill list of raw matrices with raw matrix for current mask
        masked_raw_mat_r1[mask_count] = current_raw_mat_r1
        masked_raw_mat_r2[mask_count] = current_raw_mat_r2

        # Create concatenated runs to return
        masked_raw_mat_concat[mask_count] = np.concatenate(
                [masked_raw_mat_r1[mask_count],
                masked_raw_mat_r2[mask_count]])

    # Check for concatenate variable
    # Do not concatenate runs
    if bool(not concat):
        # Return complete list of masked raw matrices in list form
        return(masked_raw_mat_r1, masked_raw_mat_r2)
    elif bool(concat):
        # Return concatenated runs
        return(masked_raw_mat_concat)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# 2. Create_beta_matrix
# Function to create input matrix for classifier training on betas
# - Define path to betas
# - Load preprocessed betas
# - Transform into input matrix (1D voxel array)
# - Return input matrix for both buffers in list of lists
#       - 1st list: buffer 1, buffer 2
#       - 2nd lst: Mask 1, ..., Mask n
#       - Example: output[0][4] = beta_mat of Buffer1, 5th Mask
#       - Example: output[1][0] = beta_mat of Buffer2, 1st Mask

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. masks: 3D array of mask values in 1s and 0s, 4D array in case of
# multiple masks
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Create_beta_matrix(bids_dir, sub_id, masks):

    # Pose input of masks in appropriate format in case only one mask is
    # given
    if len(masks.shape) == 3:
        # Add 4th dimension and fill first 3 with provided mask
        pattern = np.zeros([masks.shape[0], masks.shape[1], masks.shape[2], 1])
        pattern[:,:,:,0] = masks
        # Make new 4D array to masks
        masks = pattern

    # Define path which contains all corrected, smoothed, and z-scored  betas:
    corr_dir_b1 = (bids_dir + '/derivatives/' + sub_id +
                   '/glm/buffer1' + '/estimate/corr')
    corr_dir_b2 = (bids_dir + '/derivatives/' + sub_id +
                   '/glm/buffer2' + '/estimate/corr')

    # Give number of betas to load (skipping motion and mean related betas)
    req_betas = np.concatenate([np.arange(1,13),
                                np.arange(19,31)])

    # Allocate 4D array to store betas, dim 1-3: Voxel values, dim 4:
    # different betas
    beta_mat_b1 = np.zeros([masks.shape[0],
                            masks.shape[1],
                            masks.shape[2],
                            len(req_betas)])
    beta_mat_b2 = deepcopy(beta_mat_b1)

    # Load unmasked betas from both buffers
    for beta_count, beta_id in enumerate(req_betas):

        # Path to load specific beta
        path_b1 = corr_dir_b1 + '/zscbeta_' + str(beta_id).zfill(4) + '.nii'
        path_b2 = corr_dir_b2 + '/zscbeta_' + str(beta_id).zfill(4) + '.nii'

        # Load beta .nii with nibabel function
        nii_beta_b1 = nibabel.load(path_b1)
        nii_beta_b2 = nibabel.load(path_b2)

        # Transform nii file into matrix:
        volume_b1 = nii_beta_b1.get_data()
        volume_b2 = nii_beta_b2.get_data()

        # Concatenate beta into 4D array
        beta_mat_b1[:,:,:,beta_count] = volume_b1
        beta_mat_b2[:,:,:,beta_count] = volume_b2

    # Mask betas and pose into list with one entry for each mask
    # Create list of beta matrices with length of number of provided masks
    masked_beta_mat_b1 = [0] * masks.shape[3]
    masked_beta_mat_b2 = deepcopy(masked_beta_mat_b1)

    for mask_count in np.arange(masks.shape[3]):

        # Get current mask
        current_mask = masks[:,:,:,mask_count]

        # Allocate 2D array: 1st D holding all non-zero masked voxels, 2nd D
        # holding different betas
        current_beta_mat_b1 = np.zeros([np.flatnonzero(current_mask).size,
                                        len(req_betas)])
        current_beta_mat_b2 = deepcopy(current_beta_mat_b1)

        # Fill current beta mat with masked voxels for each beta
        for beta_count, beta_id in enumerate(req_betas):

            # Use mask to eliminate all values outside of the mask and
            # bring values into 1-D array for buffer 1 and 2
            masked_beta_b1 = beta_mat_b1[:,:,:,beta_count]
            masked_beta_b1 = masked_beta_b1[np.nonzero(current_mask)]

            masked_beta_b2 = beta_mat_b2[:,:,:,beta_count]
            masked_beta_b2 = masked_beta_b2[np.nonzero(current_mask)]

            # Fill current beta_mat
            current_beta_mat_b1[:,beta_count] = masked_beta_b1
            current_beta_mat_b2[:,beta_count] = masked_beta_b2

        # Transpose to make different betas the rows of the matrix
        current_beta_mat_b1 = np.transpose(current_beta_mat_b1, (1,0))
        current_beta_mat_b2 = np.transpose(current_beta_mat_b2, (1,0))

        # Fill list of different finished beta_mats for each mask
        masked_beta_mat_b1[mask_count] = current_beta_mat_b1
        masked_beta_mat_b2[mask_count] = current_beta_mat_b2

    # Return complete list of masked beta matrices in list form
    return(masked_beta_mat_b1, masked_beta_mat_b2)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to create 4D matrix of masks
# - Define path to mask directory
# - Load masks as nii files
# - Transform into 3D arrays
# - Append 3D arrays of masks to 4D array
# - Return 4D matrix of masks

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. mask_names: Array of strings with mask names
# 4. buffering: Logical indicating if to return one mask for each buffer or
# across buffer masks (shared mask)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Load_masks(bids_dir, sub_id, mask_names, buffering):

    # Allocate list to safe mask matrixes in
    all_masks_b1 = [0] * len(mask_names)
    all_masks_b2 = deepcopy(all_masks_b1)

    # Loop over each mask
    for mask_count, mask_id in enumerate(mask_names):

        # Get path to mask
        mask_dir = bids_dir + '/derivatives/' + sub_id + '/preproc/seg'

        # Create mask path for both buffers
        path_mask_b1 = (mask_dir + '/' + sub_id + '_mask_' + mask_id +
                        '_buffer1.nii')
        path_mask_b2 = (mask_dir + '/' + sub_id + '_mask_' + mask_id +
                        '_buffer2.nii')

        # Load mask to .nii object
        mask_b1 = nibabel.load(path_mask_b1)
        mask_b2 = nibabel.load(path_mask_b2)

        # Extract matrix from .nii
        mask_b1 = mask_b1.get_data()
        mask_b2 = mask_b2.get_data()

        # Post matrices into lists
        all_masks_b1[mask_count] = mask_b1
        all_masks_b2[mask_count] = mask_b2

    # Post list into array format with 4th D being different masks
    # Create empty 4D array with correct dimensions (given by first mask)
    all_masks_mat_b1 = np.zeros([all_masks_b1[0].shape[0],
                                 all_masks_b1[0].shape[1],
                                 all_masks_b1[0].shape[2],
                                 len(mask_names)])
    all_masks_mat_b2 = deepcopy(all_masks_mat_b1)

    # Fill 4D array with masks from lists
    for mask_count, mask_id in enumerate(mask_names):
        all_masks_mat_b1[:,:,:,mask_count] = all_masks_b1[mask_count]
        all_masks_mat_b2[:,:,:,mask_count] = all_masks_b2[mask_count]

    # Return buffer specific masks if indicated by 'buffering' parameter
    if bool(buffering):
        return(all_masks_mat_b1, all_masks_mat_b2)

    # In case combined mask is required create combined mask
    else:
        # Allocate empty 4d array holding combined masks
        all_masks_mat = np.zeros([all_masks_b1[0].shape[0],
                                 all_masks_b1[0].shape[1],
                                 all_masks_b1[0].shape[2],
                                 len(mask_names)])

        # Fill empty 4D array with combined masks
        for mask_count, mask_id in enumerate(mask_names):
            # Allocate empty mask array same shape as other masks
            mask = np.zeros([all_masks_b1[0].shape[0],
                                 all_masks_b1[0].shape[1],
                                 all_masks_b1[0].shape[2]])

            # Fill empty mask with 1 at every coordinate where both bufferd
            # masks are 1 (shared mask)
            shared_voxels = np.where(all_masks_mat_b1[:,:,:,mask_count] +
                                     all_masks_mat_b2[:,:,:,mask_count] == 2)
            mask[shared_voxels] = 1

            # Fill empty array holding all shared masks with current mask
            all_masks_mat[:,:,:,mask_count] = mask

        # Return shared masks in 4D array
        return(all_masks_mat)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to load behavioral data about runs
# - Load behavioral data about runs
# - Option to relate run1 to run2 (e.g. if last TR run 1 = 100, First TR of
# run 2 = 101)
# - Option to concatenate behavioral data
# - Add overall event counter

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. relate: Boolean if both runs should be ralted to each other (e.g. )
# 4. concat: Boolean to determine if separated runs should be returned
# concatenated or as separated runs (in list format)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Load_behavioral(bids_dir, sub_id, relate, concat):

    # Paths to both behavioral runs
    path_r1 = (bids_dir + '/derivatives/' + sub_id + '/behavior/' + sub_id +
               '_3d1_backwards.tsv')
    path_r2 = (bids_dir + '/derivatives/' + sub_id + '/behavior/' + sub_id +
               '_3d2_backwards.tsv')

    # Load behavioral .tsv files
    behav_r1 = pd.read_csv(path_r1, sep='\t')
    behav_r2 = pd.read_csv(path_r2, sep='\t')

    # If both runs should be related
    if bool(relate):

        # Make timing relative to previous timing
        behav_r2.loc[:, 'Time'] += behav_r1['Time'].iloc[-1]

        # Make TRs relative to first run
        behav_r2.loc[:, 'TR number corrected'] += (
                behav_r1['TR number uncorrected'].iloc[-1])
        behav_r2.loc[:, 'TR number uncorrected'] += (
                behav_r1['TR number uncorrected'].iloc[-1])

    # If both runs should be concatenated
    if bool(concat):

        # Concatenate both runs
        frames = [behav_r1, behav_r2]
        behav = pd.concat(frames)

        # Return concatenated runs
        return(behav)

    # If runs should be returned separately
    elif bool(not concat):
        return(behav_r1, behav_r2)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to load motion data of subject
# - Load motion data about runs stemming from realignment
# - Add overall event counter

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Load_motion(bids_dir, sub_id):

    # Paths to both behavioral runs
    path_r1 = (bids_dir + '/derivatives/' + sub_id + '/preproc/realign/' +
               sub_id + '_mov_reg1.txt')
    path_r2 = (bids_dir + '/derivatives/' + sub_id + '/preproc/realign/' +
               sub_id + '_mov_reg2.txt')

    # Load behavioral .tsv files
    motion_r1 = pd.read_csv(path_r1, delimiter=r"\s+", header=None)
    motion_r2 = pd.read_csv(path_r2, delimiter=r"\s+", header=None)

    return(motion_r1, motion_r2)

''' # Start decoding script '''

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to decode and predict direction
# - Load all data required (motion, raw, betas, behavioral)
# - Train and test classifier for each mask and save accuracy and predictions
# - Train classifier on permuted labels and save accuracy and predictions

# Parameters:
# 1. bids_dir: BIDS structured directory
# 2. sub_id: Participant code (as string)
# 3. mask_names: List of stings with mask codes
# 4. n_permutation: Number of permutations
# 5. out_dir: Output directory to save results
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def main(bids_dir, sub_id, mask_names, n_permutation, out_dir):

    #- - - - - - - - - - - - - - - - -
    # To select certain subjects:
    #sub_list = sub_list[0:1]
    #- - - - - - - - - - - - - - - - -

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Construct loop to let the classification run for separate masks
    #mask_names = ['beta',
    #              'entorhinal',
    #              'hc',
    #              'm1',
    #              'rsc',
    #              'subiculum',
    #              'thal',
    #              'v1',
    #              'ent_hc']
    # Alternative mask_names for only specific masks:
    #mask_names = ['v1', 'rsc', 'm1']

    # Df to store predictions in (1st C: Subjects, 2nd C: Mask, ...) for real
    # classification and permutation
    clf_prediction = pd.DataFrame()
    clf_prediction_permut = pd.DataFrame()
    # Df to store acuracy in
    clf_accuracy = pd.DataFrame()
    clf_accuracy_permut = pd.DataFrame()

    # Give message to user which subject is investigated
    print('------------------------------------------------------------------')
    print('Working on subject:\t\t\t--|', sub_id, '|--')
    print('------------------------------------------------------------------')

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Load required data
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Give message to user
    print('\tLoading required data...')

    # Unbuffered (shared mask array)
    mask_mat = Load_masks(bids_dir, sub_id, mask_names, buffering=0)
    # Buffered (two seperate mask arrays)
    mask_mat_b1, mask_mat_b2 = Load_masks(bids_dir, sub_id, mask_names,
                                          buffering=1)

    # Load betas for each buffer into 2D matrix (rows: betas, cols: voxels)
    beta_mat_b1, beta_mat_b2 = Create_beta_matrix(bids_dir, sub_id, mask_mat)

    # Relate betas to directions
    labels_beta = pd.DataFrame()
    labels_beta['angle_bin'] = np.tile(np.arange(1,7), 4)
    labels_beta['fold'] = np.repeat(np.arange(1,5), 6)

    # Load raw data into same format as beta matrix, masked by shared mask
    raw_mat_r1, raw_mat_r2 = Create_raw_matrix(bids_dir, sub_id, mask_mat,
                                               preproc=0,
                                               smooth=1, smooth_kernel=3,
                                               concat=0)
    raw_mat = Create_raw_matrix(bids_dir, sub_id, mask_mat,
                                           preproc=0,
                                           smooth=1, smooth_kernel=3,
                                           concat=1)

    # Load subject motion from realignment for signal.claen
    motion_r1, motion_r2 = Load_motion(bids_dir, sub_id)

    # Signal clean raw mat runs for each mask
    raw_mat_r1 = list(
            signal.clean(x,
                         confounds=motion_r1.values,
                         high_pass = 1/128,
                         t_r = 2.4)
            for x in raw_mat_r1
            )
    raw_mat_r2 = list(
            signal.clean(x,
                         confounds=motion_r2.values,
                         high_pass = 1/128,
                         t_r = 2.4)
            for x in raw_mat_r2
            )

    # Voxel-internal z-scoring
    raw_mat_r1 = list(scipy.stats.zscore(x, axis=0) for x in raw_mat_r1)
    raw_mat_r2 = list(scipy.stats.zscore(x, axis=0) for x in raw_mat_r2)

    # TR wise z-scoring
    raw_mat_r1 = list(scipy.stats.zscore(x, axis=1) for x in raw_mat_r1)
    raw_mat_r2 = list(scipy.stats.zscore(x, axis=1) for x in raw_mat_r2)

    # Load behavioral runs as concatenated file
    behav = Load_behavioral(bids_dir, sub_id, relate=1, concat=1)

    # Relate TRs to behavioral
    # Data frame holding TRs and associated direction
    labels_raw = pd.DataFrame()
    # trs of functional data (+1 since TRs in behav file are counted
    # from starting 0)
    labels_raw['tr'] = np.arange(raw_mat[0].shape[0]) + 1
    # Add column holding directional values for each TR
    labels_raw['angle_mode'] = float('nan')
    labels_raw['angle_mean'] = float('nan')

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Get directional value for each TR
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Loop over each TR with directional event (mode and mean)
    for tr_count, tr_id in enumerate(
            np.unique(behav['TR number corrected'])
            ):
        # Index of behavioral data for a certain TR
        index = behav.loc[:,'TR number corrected'] == tr_id
        # Get distribution of raw angles during TR
        angle_dist = np.round(
                behav.loc[index,'direction angle by location'].values,
                2)
        # Get mode and mean of angles during one TR
        mode_angle = scipy.stats.mode(angle_dist).mode[0]
        mean_angle = np.mean(angle_dist)

        # Fill label array with directional events (fold not needed since
        # betas don't contain backwards events we are predicting)
        # Eliminate cases in which corrected TR is beyond the collected TRs
        # (since corrected TRs is +2 TRs that might not have been measured)
        if tr_id <= raw_mat[0].shape[0]:
            index = labels_raw.loc[:,'tr'] == tr_id
            labels_raw.loc[index,'angle_mode'] = mode_angle
            labels_raw.loc[index,'angle_mean'] = np.round(mean_angle, 2)

    # Get label vector that only contains events (eliminate NaN entries)
    index = np.where(~np.isnan(labels_raw.loc[:,'angle_mean']))[0]
    predict_labels = labels_raw.loc[index,:]

    # Add bin and prediction column to label vector
    predict_labels['bin_mode'] = float('nan')
    predict_labels['bin_mean'] = float('nan')
    predict_labels['prediction'] = float('nan')

    # Fill bin columns with correct bin
    predict_labels['bin_mode'] = np.floor(
            predict_labels['angle_mode'] / 60
            ) + 1
    predict_labels['bin_mean'] = np.floor(
            predict_labels['angle_mean'] / 60
            ) + 1

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Mask specific analysis
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Loop over masks
    for mask_count, mask_id in enumerate(mask_names):

        # Give message to user: Processed mask
        print('\t', mask_id, '\t:', end='')

        # Get current mask (unbuffered and buffered)
        c_mask = mask_mat[:,:,:,mask_count]
        c_mask_b1 = mask_mat_b1[:,:,:,mask_count]
        c_mask_b2 = mask_mat_b2[:,:,:,mask_count]

        # Get current beta_mat for both buffers
        c_beta_mat_b1 = beta_mat_b1[mask_count]
        c_beta_mat_b2 = beta_mat_b2[mask_count]

        # z-score beta mat (whole single beta) for both buffers
        c_beta_mat_b1 = scipy.stats.zscore(c_beta_mat_b1, axis=1)
        c_beta_mat_b2 = scipy.stats.zscore(c_beta_mat_b2, axis=1)

        # Get current concatenated raw_mat (from shared mask)
        c_raw_mat_r1 = raw_mat_r1[mask_count]
        c_raw_mat_r2 = raw_mat_r2[mask_count]

        # Concatenate cleaned runs
        c_raw_mat = np.concatenate([c_raw_mat_r1, c_raw_mat_r2])

        # Eliminate volumes from raw_mat which can't be predicted (since TRs
        # which where in no events)
        index = predict_labels.loc[:,'tr'] - 1
        predict_raw_mat = c_raw_mat[index,:]

        # Allocate df holding accuracy of classification for each fold
        results_accuracy = pd.DataFrame()
        results_accuracy['accuracy_mean'] = float('nan')
        results_accuracy['accuracy_mode'] = float('nan')
        results_accuracy['correlation_mean'] = float('nan')
        results_accuracy['correlation_mode'] = float('nan')

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Start classification
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        # Give message to user: Classification starts
        print('\tClassification...', end='')

        # Classification of backwards events (no corss valitation needed since
        # backwards events are no in buffers)

        # Get training set and trinaing labels for fold (over concatenated
        # buffers of betas)
        train_set = np.concatenate([c_beta_mat_b1, c_beta_mat_b2])
        train_labels = np.concatenate([labels_beta.loc[:,'angle_bin'].values,
                                       labels_beta.loc[:,'angle_bin'].values])

        # Get testing set and testing labels (over concatenated and cleaned
        # raw_data)
        test_set = predict_raw_mat

        # Labels and bins for mode
        test_labels_mode = predict_labels.loc[:, 'angle_mode'].values
        test_bins_mode = predict_labels.loc[:, 'bin_mode'].values
        # Labels and bins for mean
        test_labels_mean = predict_labels.loc[:, 'angle_mean'].values
        test_bins_mean = predict_labels.loc[:, 'bin_mean'].values

        # Create classifier object
        clf = sklearn.svm.LinearSVC(C=1,
                                    max_iter=10000)

        # Train classifier
        clf.fit(train_set, train_labels)

        # Predict angle_bins
        prediction = clf.predict(test_set)

        # Put prediction into label vector
        predict_labels.loc[:,'prediction'] = prediction

        # Save accuracy for mean and mode by comparing number of correct
        # predictions of bin to overall number of predictions
        # Mean
        results_accuracy.loc[0,'accuracy_mean'] = (
                np.sum(np.equal(prediction, test_bins_mean)) /
                len(prediction)
                )
        # Mode
        results_accuracy.loc[0,'accuracy_mode'] = (
                np.sum(np.equal(prediction, test_bins_mode)) /
                len(prediction)
                )

        # Save correlation between predicted bin and continuous label
        # Mean
        results_accuracy.loc[0,'correlation_mean'] = (
                scipy.stats.pearsonr(prediction, test_labels_mean)[0]
                )
        # Mode
        results_accuracy.loc[0,'correlation_mode'] = (
                scipy.stats.pearsonr(prediction, test_labels_mode)[0]
                )

        # Save label vector with predictions into data frame
        results_prediction = deepcopy(predict_labels)
        results_prediction['sub_id'] = sub_id
        results_prediction['mask'] = mask_id

        # Add subject id and mask to data frame of accuracy
        results_accuracy['sub_id']= sub_id
        results_accuracy['mask']= mask_id

        # Append to results data frame
        clf_prediction = clf_prediction.append(results_prediction)
        clf_accuracy = clf_accuracy.append(results_accuracy)

        # Give message to user: Classification done
        print('Done | ', end='')

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Start permutation
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        # Give message to user: Permutation starts
        print('Permutation...', end='')

        # Same classification process with permuted labels
        for perm_count in np.arange(n_permutation):

            # Get training set and training labels for fold (over
            # concatenated buffers of betas)
            train_set = np.concatenate([c_beta_mat_b1, c_beta_mat_b2])

            # Shuffle labels inside each fold per buffer
            # Loop over cross validation folds (iterated fold is testing fold)
            train_labels = []
            for fold_count in np.arange(8):
                shuffled_labels = sample(list(np.arange(1,7)), k=6)
                train_labels = np.concatenate([train_labels, shuffled_labels])

            # Get testing set and testing labels (over concatenated and
            # cleaned raw_data) (test labels stay unshuffled)
            test_set = predict_raw_mat
            # Labels and bins for mode
            test_labels_mode = predict_labels.loc[:, 'angle_mode'].values
            test_bins_mode = predict_labels.loc[:, 'bin_mode'].values
            # Labels and bins for mean
            test_labels_mean = predict_labels.loc[:, 'angle_mean'].values
            test_bins_mean = predict_labels.loc[:, 'bin_mean'].values

            # Create classifier object
            clf = sklearn.svm.LinearSVC(C=1,
                                        max_iter=10000)

            # Train classifier
            clf.fit(train_set, train_labels)

            # Predict angle_bins
            prediction = clf.predict(test_set)

            # Put prediction into label vector
            predict_labels.loc[:,'prediction'] = prediction

            # Save accuracy for mean and mode by comparing number of
            # correct predictions of bin to overall number of predictions
            # Mean
            results_accuracy.loc[:,'accuracy_mean'] = (
                    np.sum(np.equal(prediction, test_bins_mean)) /
                    len(prediction)
                    )
            # Mode
            results_accuracy.loc[:,'accuracy_mode'] = (
                    np.sum(np.equal(prediction, test_bins_mode)) /
                    len(prediction)
                    )

            # Save correlation between predicted bin and continuous label
            # Mean
            results_accuracy.loc[:,'correlation_mean'] = (
                    scipy.stats.pearsonr(prediction, test_labels_mean)[0]
                    )
            # Mode
            results_accuracy.loc[:,'correlation_mode'] = (
                    scipy.stats.pearsonr(prediction, test_labels_mode)[0]
                    )

            # Save label vector with predictions into data frame
            results_prediction = deepcopy(predict_labels)
            results_prediction['sub_id'] = sub_id
            results_prediction['mask'] = mask_id
            results_prediction['permutation'] = perm_count

            # Add subject id and mask to data frame of accuracy
            results_accuracy['sub_id']= sub_id
            results_accuracy['mask']= mask_id
            results_accuracy['permutation']= perm_count

            # Append to results data frame
            clf_prediction_permut = clf_prediction_permut.append(
                    results_prediction)
            clf_accuracy_permut = clf_accuracy_permut.append(results_accuracy)

        # Give message to user: Permutation done
        print('Done')

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Save results for subject
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Sort results DFs and save to .tsv
    # Prediction
    clf_prediction = clf_prediction[['sub_id', 'mask',
                                     'tr',
                                     'angle_mode', 'angle_mean',
                                     'bin_mode', 'bin_mean',
                                     'prediction']]
    path = out_dir + '/' + sub_id + '_clf_prediction.tsv'
    clf_prediction.to_csv(path, sep='\t', index=False)
    # Accuracy
    clf_accuracy = clf_accuracy[['sub_id', 'mask',
                                 'accuracy_mode', 'accuracy_mean',
                                 'correlation_mode', 'correlation_mean']]
    path = out_dir + '/' + sub_id + '_clf_accuracy.tsv'
    clf_accuracy.to_csv(path, sep='\t', index=False)

    # Permutation files
    # Prediction
    clf_prediction_permut = clf_prediction_permut[['sub_id', 'mask',
                                                   'tr',
                                                   'angle_mode',
                                                   'angle_mean',
                                                   'bin_mode', 'bin_mean',
                                                   'prediction',
                                                   'permutation']]
    path = out_dir + '/' + sub_id + '_clf_prediction_permut.tsv'
    clf_prediction_permut.to_csv(path, sep='\t', index=False)
    # Accuracy
    clf_accuracy_permut = clf_accuracy_permut[['sub_id', 'mask',
                                               'accuracy_mode',
                                               'accuracy_mean',
                                               'correlation_mode',
                                               'correlation_mean',
                                               'permutation']]
    path = out_dir + '/' + sub_id + '_clf_accuracy_permut.tsv'
    clf_accuracy_permut.to_csv(path, sep='\t', index=False)

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Basic plotting
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

#    for mask_count, mask_id in enumerate(mask_names):
#
#        fig = plt.figure(figsize=(8,8))
#        gs = gridspec.GridSpec(2,3, wspace=0.3, hspace=0.2)
#
#        mask_results = clf_prediction.loc[clf_prediction['mask'] == mask_id,:]
#
#        for bin_count, bin_id in enumerate(np.arange(1,7)):
#
#            pred_dist = mask_results.loc[mask_results['bin_mode'] == bin_id,
#                                         'prediction']
#
#            ax = fig.add_subplot(gs[bin_count])
#            ax.hist(pred_dist, bins=np.arange(1,8), density=True, rwidth=0.8)
#            ax.set_xticks(np.arange(1,7)+0.5)
#            ax.set_xticklabels(np.arange(1,7))
#            ax.set_ylim(bottom=0, top=1)
#            ax.set_title('Direction ' + str(bin_id), weight='bold')
#            fig.add_subplot(ax)
#        plt.show()

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------

''' Decoding for subject depending on input'''

# Set up paths
# BIDS directory (containing betas and raw data)
bids_dir = str(Path.home()) + '/direction_decoding_BIDS'
# Path of repository
repo_dir = str(Path.home() + '/direction_decoding')
# Path to save results to (relative to repo)
save_dir = (repo_dir + '/decoding/backwards_decoding/train_beta_test_raw' +
            '/results')

# Define key variables based on command line input
bids_dir = argv[1]
sub_id = argv[2]
mask_names = argv[3]
# Convert input to list
mask_names = mask_names.strip('[]').split(',')
n_permutation = int(argv[4])
out_dir = argv[5]

# Call decoding function with user inputs
main(bids_dir, sub_id, mask_names, n_permutation, out_dir)


#!/usr/bin/env python3

import sys
import collections
import itertools
import numpy

import aoc

infilename = 'input.txt'
if len(sys.argv) > 2 and sys.argv[1] == '-i':
    infilename = sys.argv[2]

print('Using input file: %s' % infilename)

f = open(infilename, 'r')
data = f.readlines()
f.close()

data = [line.strip() for line in data]

scanners = {}
for ii in range(len(data)):
    line = data[ii]
    if not line:
        continue

    if line.startswith('--'):
        curr = int(line.split()[2])
        scanners[curr] = []
        continue

    scanners[curr].append(list(map(int, line.split(','))))

# compute manhattan distances from each sensor node to all other nodes, compare to
# each sensor and find sets that have 12 of the same distances which are then the
# shared nodes between two sensors
matches=[]
for mm in range(len(scanners)):
    for nn in range(mm+1, len(scanners)):
        if mm == nn:
            continue
        for xx in range(len(scanners[mm])):
            for yy in range(len(scanners[nn])):
                dists0 = [aoc.mandist(scanners[mm][xx], pos) for pos in scanners[mm]]
                dists1 = [aoc.mandist(scanners[nn][yy], pos) for pos in scanners[nn]]
                s = set(dists0).intersection(set(dists1))
                if len(s)>=12:
                    print(mm, nn,xx,yy, s)
                    matches.append((mm, nn, xx, yy))
                    matches.append((nn, mm, yy, xx))

def translate_all(scanners, matches):
    open_set = list(range(len(scanners)))
    open_set.remove(0)
    closed_set = [0]

    beacons = [tuple(t) for t in scanners[0]]

    match_pairs = list(set([(el[0], el[1]) for el in matches]))

    translated = {}
    offsets = {0:0}

    sensor_pos = [(0,0,0)]

    iters = 0
    while open_set:

        # get the next mapping to do - any pair that matches a closed sensor
        # to an open sensor
        pair = None
        for p in match_pairs:
            if p[0] in closed_set and p[1] in open_set:
                pair = p
                break
        assert(pair)

        left = pair[0]
        right = pair[1]
        curr_batch = [el for el in matches if el[0] == left and el[1] == right]

        done = False
        # figure out which orientation results in all of the shared nodes point to the same
        # sensor pos
        for inds in itertools.permutations(range(3),3):
            for signs in itertools.product([-1,1], repeat=3):
                inds = numpy.array(inds)
                signs = numpy.array(signs)

                # found when all shared nodes rotate and offset to the same position relative to sensor 0
                pos = [aoc.tup_add(beacons[offsets[left]+el[2]], numpy.array(scanners[right][el[3]])[inds] * numpy.array(signs)) for el in curr_batch]
                if len(set(pos))==1:
                    #print(pos)
                    sensor_pos.append(pos[0])
                    done = True
                    break
            if done:
                break

        assert(done)

        # translate all of the beacons to the sensor 0 space (shared plus nonshared) and
        # track the offsets so our original mapping from above still works
        offsets[right] = len(beacons)
        trans = [aoc.tup_sub(pos[0], numpy.array(el)[inds] * numpy.array(signs)) for el in scanners[right]]
        beacons.extend(trans)
        translated[right] = trans

        iters += 1

        open_set.remove(right)
        closed_set.append(right)

    return beacons, sensor_pos

all_beacons, sensors = translate_all(scanners, matches)
print(len(sensors), sensors)
print(len(set(all_beacons)))

maxd = 0
for xx, yy in itertools.combinations(range(len(sensors)),2):
    d = aoc.mandist(sensors[xx], sensors[yy])
    if d > maxd:
        maxd = d
print(maxd)


import logging, fastaResFun
from Bio import SeqIO
import json, sys

def isoformAln(aln, o):
        """Function to cluster isoforms according to the alignment. Return the
                overall coverage of these isoforms.

        Isoforms are from the same species (recognized through keyword
        xxxXxx at the beginning of their name) and same letters or
        indels at same positions in alignment.

        @param1 aln: Path to alignment
        @param2 o: output directory
        @return outAln: Path to file of resulting alignment

        """

        logger = logging.getLogger("main.alignment")
        print("Clustering isoforms.")

        dRem={} #for remaining sequences
        dId2Seq={} #for remaining sequences
        laln=0 #alignement length
        for fasta in SeqIO.parse(open(aln),'fasta'):
                post=fasta.id.find("_")
                if post!=-1: #regular format
                        sp=fasta.id[:post]
                        tag=fasta.id[post+1:]
                        if not sp in dId2Seq:
                                dId2Seq[sp]={}
                        dId2Seq[sp][tag]=str(fasta.seq)
                        if laln==0:
                                laln=len(fasta.seq)
                else:
                        dRem[fasta.id]=str(fasta.seq)

        outCov = o
        clustok=False #flag to check if a cluster has occured
        for sp,dtagseq in dId2Seq.items():
                lclust=[list(dtagseq)] #list of clusters of tags to be split
                for pos in range(laln):
                        lclust2=[]
                        for clust in lclust:
                                dlet={tag:dtagseq[tag][pos] for tag in clust}
                                llet=set([x for x in dlet.values() if x!="-"])
                                if len(llet)<=1: #one letter at most, keep all
                                        lclust2.append(clust)
                                        continue
                                else:
                                        for x in llet:
                                                lclust2.append([tag for tag in clust if dlet[tag]==x])
                                        lind=[tag for tag in clust if dlet[tag]=="-"] #conservative, do not know wether to merge, may be improved
                                        if len(lind)!=0:
                                                lclust2.append(lind)
                        lclust=lclust2

                #now merge sequences in each cluster
                for clust in lclust:
                        if len(clust)==1:
                                dRem[sp+"_"+clust[0]]=dtagseq[clust[0]]
                        else:
                                clustok=True
                                ntag=clust[-1]+"_clust"
                                print("Clustered sequences " + sp+"_" + (", %s_"%(sp)).join(clust) + " into %s_"%(sp)+ntag)
                                nseq="".join([max([dtagseq[tag][pos] for tag in clust]) for pos in range(laln)])
                                dRem[sp+"_"+ntag]=nseq

        if clustok:
            with open(outCov, "w") as outC:
                outC.write(fastaResFun.dict2fasta(dRem))
                outC.close()

                return(outCov)
        else:
                return(aln)


# -*- coding: utf-8 -*-
from distutils.core import setup
from setuptools import find_packages
import re
from os import path

v_file = open(path.join(path.dirname(__file__),
                        'ceda_markup', '__init__.py'))
VERSION = re.compile(r".*__version__ = '(.*?)'",
                     re.S).match(v_file.read()).group(1)

setup(
    name='ceda-markup',
    version=VERSION,
    author=u'Maurizio Nagni',
    author_email='maurizio.nagni',
    packages=find_packages(),
    url='http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/CedaMarkup',
    license='BSD licence, see LICENCE',
    description='Collection of markup classes as geosrss, gml, atom, rss...' + \
                ' Contains an OpenSearch server (just the core not the server)',
    long_description=open('README').read(),
    zip_safe=False,
)

'''
zip_safe=False option. It prevents the package manager to install a
      python egg, instead you'll get a real directory with files in it.
'''


import wx
import time

def LongSimulation(self,input_):
    # Simulate the behavior of the simulation code that I have
    # Actually it returns more than just the progress
    # Occasionally it also returns some partial summary result for diagnostic
    for i in range(0, 10):
        self.log.write('Progress: %3.0f\r' % float(i))
        time.sleep(1)
        wx.GetApp().Yield()
    return 'answer'

class MyForm(wx.Frame):

    def __init__(self):
        wx.Frame.__init__(self, None)

        # Add a panel so it looks the correct on all platforms
        panel = wx.Panel(self, wx.ID_ANY)
        style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL

        sizer = wx.BoxSizer(wx.VERTICAL)

        self.log = wx.TextCtrl(panel, wx.ID_ANY, size=(300,100), style=style)
        sizer.Add(self.log, 1, wx.ALL|wx.EXPAND, 5)

        btn = wx.Button(panel, wx.ID_ANY, 'Start')
        self.Bind(wx.EVT_BUTTON, self.onButton, btn)

        sizer.Add(btn, 0, wx.ALL|wx.CENTER, 5)
        panel.SetSizer(sizer)

    def onButton(self, event):
        input_ = 0
        res = LongSimulation(self,input_)
        self.log.write(res)
        self.log.write('\n2222')

# Run the program
if __name__ == "__main__":
    app = wx.App(False)
    frame = MyForm()
    frame.Show()
    app.MainLoop()