import lxml.html
import datetime
import dateutil.relativedelta
import logging
import requests
import os
import tarfile
from io import BytesIO

class WikipediaLogDownloader:

    def __init__(self, folder=""):
        self.folder = folder
        if not os.path.exists(folder):
                os.makedirs(folder)
        self.log = logging.getLogger(__name__)
        logging.getLogger("requests").setLevel(logging.WARNING)

    def download_file(self, file_url, file_date, hour, tgz=None):
        r = requests.get(file_url)
        if r.status_code != 200:
            self.log.error("Can't get file %s", file_url)
            return

        if self.folder != "":
            folder = self.folder + "/"
        else:
            folder = self.folder

        if tgz is None:
            try:
                with open(folder + str(file_date) +
                          "_" + hour + ".log", "w") as f:
                    f.write(r.text)
            except:
                self.log.error("Error for file " + file_url)
                with open(folder + "error_log.txt", "a") as ef:
                    ef.write("Error for file " + file_url)
                return
        else:
            try:
                info = tarfile.TarInfo(str(file_date) + "_" + hour + ".log")
                info.size = len(r.content)
                tgz.addfile(info, BytesIO(bytes(r.content)))
            except:
                self.log.error("Error for file " + file_url)
                with open(folder + "error_log.txt", "a") as ef:
                    ef.write("Error for file " + file_url)
                return

    def obsolete_download_month(self, current_date, start_date, end_date):
        '''Downloads all the files for the month of current_date as long as they are
        before end_date. Writes all of them in a single .tgz file.
        All dates are datetime.date objects.
        '''
        base_url = 'http://dumps.wikimedia.org/other/pagecounts-raw'

        year = str(current_date.year)
        month = "{:02d}".format(current_date.month)

        index_url = base_url + '/' + year + '/' + year + "-" + month

        dom = lxml.html.fromstring(requests.get(index_url).text)

        tgz = tarfile.open(self.folder + "/" +
                           datetime.date(int(year), int(month)) +
                           ".tgz", mode="w:bz2")
        for link in dom.xpath('//a/@href'):
            # select the url in href for all a tags
            if (link.startswith('projectcounts')):
                hour = link[-6:-4]
                day = link[-9:-7]
                file_url = index_url + '/' + link
                file_date = datetime.date(int(year), int(month), int(day))
                if start_date <= file_date < end_date:
                    self.download_file(file_url, file_date, hour, tgz=tgz)
        tgz.close()

    def download_month(self, year, month):
        '''Downloads all the files for the month of current_date as long as they are
        before end_date. Writes all of them in a single .tgz file.
        All dates are datetime.date objects.
        '''
        base_url = 'http://dumps.wikimedia.org/other/pagecounts-raw'

        start_date = datetime.date(year, month, 1)
        end_date = start_date + dateutil.relativedelta.relativedelta(months=+1)
        year = str(year)
        month = "{:02d}".format(month)

        index_url = base_url + '/' + year + '/' + year + "-" + month

        dom = lxml.html.fromstring(requests.get(index_url).text)

        tgz = tarfile.open("{}/{}-{}.tbz".format(self.folder, year, month),
                           mode="w:bz2")
        for link in dom.xpath('//a/@href'):
            # select the url in href for all a tags
            if (link.startswith('projectcounts')):
                hour = link[-6:-4]
                day = link[-9:-7]
                file_url = index_url + '/' + link
                file_date = datetime.date(int(year), int(month), int(day))
                if start_date <= file_date < end_date:
                    self.download_file(file_url, file_date, hour, tgz=tgz)
        tgz.close()

    def download(self, start_year, start_month, start_day,
                 number_of_days, folder=""):
        start_date = datetime.date(start_year, start_month, start_day)
        end_date = start_date + datetime.timedelta(days=number_of_days)

        self.folder = folder

        # Iterate over the months, as there is one index page per month
        current_date = start_date
        while current_date < end_date:
            self.download_month(current_date, start_date, end_date)
            current_date = (current_date +
                            dateutil.relativedelta.relativedelta(months=+1))


# -*- coding: utf-8 -*-
# @Time    : 2018/5/21 9:07
# @Author  : lanxin yang
# @FileName: augmentation.py

import cv2
import numpy as np
from utils.config import *

def gamma_transform(img,gamma_vari):
    """
    Gamma变换： y=x^gamma;
    通过非线性变换让图像从对曝光强度的线性响应变得更接近人眼感受到的响应
    gamma>1, 拉伸图像中灰度级较低的区域，同时压缩灰度级较高的部分，图像整体变暗；
    gamma<1, 拉伸图像中灰度级较高的区域，同时压缩灰度级较低的部分，图像整体变亮；
    """
    log_gamma_vari = np.log(gamma_vari)
    alpha = np.random.uniform(-log_gamma_vari, log_gamma_vari)
    gamma = np.exp(alpha)

    gamma_table=[np.power(x/255.0,gamma)*255.0 for x in range(256)]
    gamma_table=np.round(np.array(gamma_table)).astype(np.uint8)

    return cv2.LUT(img,gamma_table)

def rotate(img,lab,angle):
    """
    旋转操作
    """
    M_rotate = cv2.getRotationMatrix2D((IMG_W/2, IMG_H/2), angle, 1)
    img = cv2.warpAffine(img, M_rotate, (IMG_W, IMG_H))
    lab = cv2.warpAffine(lab, M_rotate, (IMG_W, IMG_H))
    return img,lab

def blur(img):
    """
    用低通滤波来平滑图像
    """
    img=cv2.blur(img,(3,3))
    return img

def add_noise(img):
    """
    加噪操作
    """
    for i in range(200):
        temp_x = np.random.randint(0,img.shape[0])
        temp_y = np.random.randint(0,img.shape[1])
        img[temp_x][temp_y] = 255
    return img

def data_augment(img, lab):
    """
    增强操作
    """
    if np.random.random() < 0.25:
        img, yb = rotate(img, lab, 90)
    if np.random.random() < 0.25:
        img, yb = rotate(img, lab, 180)
    if np.random.random() < 0.25:
        img, yb = rotate(img, lab, 270)
    if np.random.random() < 0.25:
        img = cv2.flip(img, 1)  # flipcode > 0：沿y轴翻转
        lab = cv2.flip(lab, 1)

    if np.random.random() < 0.25:
        img = gamma_transform(img, 1.0)

    if np.random.random() < 0.25:
        img = blur(img)

    if np.random.random() < 0.2:
        img = add_noise(img)

    return img, lab


"""Read analytics from local SQL files."""
from os import listdir
from os.path import isfile, join
from typing import List

from sqlalchemy.engine.result import ResultProxy

from config import basedir
from database import rdbms

def collect_sql_queries(subdirectory: str) -> dict:
    """
    Create dict of SQL queries to be run where `keys` are filenames and `values` are queries.

    :param subdirectory: Directory containing .sql queries to run in bulk.
    :type subdirectory: str
    :returns: dict
    """
    sql_file_paths = fetch_sql_files(subdirectory)
    sql_queries = parse_sql_batch(sql_file_paths)
    sql_file_names = [file.split("/")[-1] for file in sql_file_paths]
    query_dict = dict(zip(sql_file_names, sql_queries))
    return query_dict

def fetch_sql_files(subdirectory: str) -> List[str]:
    """
    Fetch all SQL query files in folder.

    :param subdirectory: Subdirectory containing SQL files to fetch.
    :type subdirectory: str
    :returns: List[str]
    """
    folder = f"{basedir}/database/queries/{subdirectory}"
    directory = listdir(folder)
    files = [
        folder + "/" + f for f in directory if isfile(join(folder, f)) if ".sql" in f
    ]
    return files

def parse_sql_batch(sql_file_paths: List[str]) -> List[str]:
    """
    Read SQL analytics from .sql files.

    :param sql_file_paths: List of paths to SQL files to read and parse.
    :type sql_file_paths: List[str]
    :returns: List[str]
    """
    queries = []
    for file in sql_file_paths:
        sql_file = open(file, "r")
        query = sql_file.read()
        queries.append(query)
        sql_file.close()
    return queries

def fetch_raw_lynx_posts() -> ResultProxy:
    """
    Find all Lynx posts lacking embedded link previews.

    :returns: ResultProxy
    """
    sql_file = open(f"{basedir}/database/queries/posts/selects/lynx_bookmarks.sql", "r")
    query = sql_file.read()
    posts = rdbms.execute_query(query, "hackers_prod").fetchall()
    return posts


# coding=utf-8
# --------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
#
# Code generated by Microsoft (R) AutoRest Code Generator 2.3.33.0
# Changes may cause incorrect behavior and will be lost if the code is
# regenerated.
# --------------------------------------------------------------------------

from .optimization_config import OptimizationConfig

class ModelOptimizationRequestOptimizationConfig(OptimizationConfig):
    """the optimization configuration.

    :param inputs_schema: The inputs schema used to convert and optimize
     model.
    :type inputs_schema: list[~_restclient.models.ModelSchema]
    :param outputs_schema: The outputs schema used to convert and optimize
     model.
    :type outputs_schema: list[~_restclient.models.ModelSchema]
    :param onnx_tuning_enabled: Whether enable the onnx performance tuning.
    :type onnx_tuning_enabled: bool
    :param model_path: The relative model file/folder path in the registered
     model artifacts.
    :type model_path: str
    :param sample_input_data: Sample Input Data for the Model. A reference to
     a dataset in the workspace in the format aml://dataset/{datasetId}
    :type sample_input_data: str
    :param framework: Constant filled by server.
    :type framework: str
    """

    _validation = {
        'onnx_tuning_enabled': {'required': True},
        'framework': {'required': True},
    }

    def __init__(self, onnx_tuning_enabled, inputs_schema=None, outputs_schema=None, model_path=None, sample_input_data=None):
        super(ModelOptimizationRequestOptimizationConfig, self).__init__(inputs_schema=inputs_schema, outputs_schema=outputs_schema, onnx_tuning_enabled=onnx_tuning_enabled, model_path=model_path, sample_input_data=sample_input_data)
        self.framework = 'ModelOptimizationRequest_optimizationConfig'


import sys
sys.path.append('..')

from models.base_class import Model
from activations import relu
from layers import conv2d, batch_normalization, fully_connected, max_pooling, avg_pooling, global_avg_pooling
import tensorflow.compat.v1 as tf

class SKResNeXt(Model):
    STRUCTURES = {
        29: {
            'block_nums': [3, 3, 3],
            'cardinality': 16
        }
    }

    FIRST_CONV_OUT_CHANNEL =  64
    BOTTLENECK_OUT_CHANNEL = 4
    BOTTLENECK_CHANNEL_EXPANSION = 4
    CONV_KERNEL = 3
    REDUCTION = 16
    L = 32
    M = [3, 1]

    def __init__(self, depth, class_num, is_small):
        self.depth = depth
        self.class_num = class_num
        self.is_small = is_small

        self.structure = self.STRUCTURES[self.depth]['block_nums']
        self.cardinality = self.STRUCTURES[self.depth]['cardinality']

    def sk_module(self, x, out_channel, stride, name):
        with tf.variable_scope(name):
            u_list = list()
            with tf.variable_scope('split'):
                # 对同一输入使用不同大小的卷积核完成卷积
                for idx, k in enumerate(self.M):
                    u_list.append(
                        conv2d(x,
                               out_channel,
                               kernel=k,
                               stride=stride,
                               padding='SAME',
                               name='conv_{}'.format(idx))
                    )
                # 将所有不同尺度的特征连接成一个张量
                # u_list形状为[num_fea, batch_size, H', W', C_out]
                u_list = tf.stack(u_list, axis=0)

            with tf.variable_scope('fuse'):
                # 把所有不同尺度的特征按位加得到融合特征
                # u的形状为[batch_size, H', W', C_out]
                u = tf.reduce_sum(u_list, axis=0, name='sum')

                # 将融合的特征进行压缩
                s = global_avg_pooling(u, name='global_avg_pool')

                # 计算全连接层输出的特征数
                fc_out = max(out_channel // self.REDUCTION, self.L)
                z = fully_connected(s,
                                    fc_out,
                                    name='fully_connected')
                z = relu(z, name='relu')

            attention_list = list()
            with tf.variable_scope('select'):
                # 对每一个不同尺度的特征计算相应的权重
                for idx in range(len(self.M)):
                    attention_list.append(
                        fully_connected(z,
                                        out_channel,
                                        name='fully_connected_{}'.format(idx))
                    )
                # 将所有不同尺度特征的权重连接成一个张量
                # attention_list形状为[num_fea, batch_size, C_out]
                attention_list = tf.stack(attention_list, axis=0)

                # 不同尺度特征的权重之间使用Softmax互相抑制
                attention_list = tf.math.softmax(attention_list,
                                                 axis=0,
                                                 name='softmax')

                # 为了方便按位乘，将其形状变为[num_fea, batch_size, 1, 1, C_out]
                attention_list = tf.expand_dims(attention_list, axis=2)
                attention_list = tf.expand_dims(attention_list, axis=2)

                # 使用特征与其对应的权重按位乘
                # output形状为[num_fea, batch_size, H', W', C_out]
                output = u_list * attention_list

                # 将按位乘的结果按位加进行融合
                # output形状为[batch_size, H', W', C_out]
                output = tf.reduce_sum(output, axis=0, name='merge')

            return output

    def bottleneck(self, x, out_channel, stride, name):
        def transform(x, out_channel, stride):
            x_list = tf.split(x,
                              num_or_size_splits=self.cardinality,
                              axis=-1, name='split')
            out_channel = out_channel // self.cardinality

            # 将ResNeXt中的Bottleneck的分组卷积操作全部替换为SK模块
            for idx, x in enumerate(x_list):
                with tf.variable_scope('group_conv_{}'.format(idx)):
                    x = self.sk_module(x,
                                       out_channel,
                                       stride=stride,
                                       name='sk_module_{}'.format(idx))
                    x = batch_normalization(x,
                                            name='bn',
                                            is_training=self.is_training)
                    x = relu(x, name='relu')
                    out_x = x if idx == 0 else tf.concat([out_x, x], axis=-1)

            return out_x

        with tf.variable_scope(name):
            # 第1个1*1卷积
            output = conv2d(x,
                            out_channel,
                            kernel=1,
                            stride=1, name='conv1')
            output = batch_normalization(output,
                                         name='bn1',
                                         is_training=self.is_training)
            output = relu(output, name='relu')

            # SK模块
            with tf.variable_scope('transform'):
                output = transform(output,
                                   out_channel=out_channel,
                                   stride=stride)

            # 第3个1*1卷积
            output = conv2d(output,
                            out_channel * self.BOTTLENECK_CHANNEL_EXPANSION,
                            kernel=1,
                            stride=1,
                            padding='SAME', name='conv2')
            output = batch_normalization(output,
                                         name='bn2',
                                         is_training=self.is_training)

            c_in = x.get_shape().as_list()[-1]
            c_out = out_channel * self.BOTTLENECK_CHANNEL_EXPANSION

            with tf.variable_scope('shortcut'):
                if stride != 1 or c_in != c_out:
                    shortcut = conv2d(x,
                                      out_channel * self.BOTTLENECK_CHANNEL_EXPANSION,
                                      kernel=1,
                                      stride=stride,
                                      padding='SAME', name='conv')
                    shortcut = batch_normalization(shortcut,
                                                   name='bn',
                                                   is_training=self.is_training)
                else:
                    shortcut = x

            output = output + shortcut
            output = relu(output, name='relu')

            return output

    def build(self, x, is_training):
        self.is_training = is_training

        with tf.variable_scope('skresnext_{}_{}x{}d'.format(self.depth, self.cardinality, self.BOTTLENECK_OUT_CHANNEL)):
            with tf.variable_scope('preprocess_layers'):
                x = conv2d(x, self.FIRST_CONV_OUT_CHANNEL, kernel=3, stride=1, padding='SAME', name='conv')
                x = batch_normalization(x, name='bn', is_training=self.is_training)
                x = relu(x, name='relu')

            for idx, st in enumerate(self.structure):
                out_channel = self.cardinality * self.BOTTLENECK_OUT_CHANNEL * 2 ** idx

                if idx == 0:
                    first_stride = 1
                else:
                    first_stride = 2

                strides = [first_stride, *([1] * (st - 1))]

                for i, stride in zip(range(st), strides):
                    x = self.bottleneck(x, out_channel=out_channel, stride=stride, name='block_{}_{}'.format(idx, i))

            with tf.variable_scope('postprocess_layers'):
                if self.is_small:
                    x = global_avg_pooling(x, name='global_avg_pool')
                else:
                    x = avg_pooling(x, kernel=4, stride=4, name='avg_pool')

            with tf.variable_scope('classifier'):
                x = fully_connected(x, self.class_num, name='fully_connected')

            return x

if __name__ == "__main__":
    class_num = 10
    batch_size = None
    image = tf.placeholder(dtype=tf.float32, shape=[batch_size, 32, 32, 3])

    # model = SKResNeXt(29, class_num=class_num, is_small=False)

    # sk_out = model.sk_module(image, 4096, name='test_sk')
    # print(sk_out)
    # exit(0)

    for d in (29, ):
        model = SKResNeXt(d, class_num=class_num, is_small=False)
        output = model.build(image, is_training=False)

        print((batch_size, class_num), output.shape)

    from tools import print_net_info
    print_net_info()


from tracer import logger
import json

class display:

    def __init__(self, printAll, jsonFormat):
        '''
        Initializes the attributes of the packetDict object

        Starts the packet dict with an empty dictionary

        The constructor for the packetDict class
        takes a packet object and a fieldName
        The field name is the first line of the header
        which details the HTTP method and URL

        Parameters:

        boolean printAll - If True, print all intercepted HTTP packets
        boolean jsonFormat - If True, print to stdout in newline-separated JSON objects
                             If False, print to stdout in pretty-printed format
                             Pretty-printed format uses newline-separated key-value pairs
                             whose keys are separated from their values by a colon and whitespace.

        Returns: packetDict
        '''
        self.printAll = printAll
        self.json = jsonFormat

        access_token         = 'access_token'
        access_token_expiry  = 'expires_in'
        refresh_token        = 'refresh_token'
        refresh_token_expiry = 'refresh_expires_in'
        token_type           = 'token_type'
        id_token             = 'id_token'
        not_before_policy    = 'not-before-policy'
        session_state        = 'session_state'

        packetSize           = 'packetSize'
        sourceIP             = 'sourceIP'
        destIP               = 'destIP'
        destPort             = 'destPort'
        sourcePort           = 'sourcePort'
        clientSecret         = 'clientSecret'
        grantType            = 'grantType'
        clientId             = 'clientId'
        httpHeader           = 'httpHeader'
        refreshToken         = 'refreshToken'
        authorizationCode    = 'authorizationCode'
        redirectUri          = 'redirectUri'
        scope                = 'scope'
        accessToken          = 'accessToken'
        accessTokenExpiry    = 'accessTokenExpiry'
        refreshToken         = 'refreshToken'
        refreshTokenExpiry   = 'refreshTokenExpiry'
        tokenType            = 'tokenType'
        idToken              = 'idToken'
        timestamp            = 'timestamp'

        # printList defines the default sequence to print
        # printList is a list of doubles
        # the first entry of a double is the pretty-printed name
        # and the second entry is the key to access the pDict for the value
        # keys that dont have entries are ignored in the pDict
        self.printList = [("Timestamp", timestamp),
                          ("Transmission Time", "http_time"),
                          ("Packet Size", packetSize),
                          ("Date", "http_date"),
                          ("Cookie", "http_cookie"),
                          ("User Agent", "http_user_agent"),
                          ("Connection", "http_connection"),
                          ("Request Full URI", "http_request_full_uri"),
                          ("Request Method", "http_request_method"),
                          ("Accept Language", "http_accept_language"),
                          ("Chat", "http_chat"),
                          ("Content Type", "content_type"),
                          ("Response Code", "response_code"),
                          ("Response Phrase", "response_phrase"),
                          ("Source IP", sourceIP),
                          ("Source Port", sourcePort),
                          ("Destination IP", destIP),
                          ("Destination Port", destPort),
                          ("Server", "server"),
                          ("Client Secret", clientSecret),
                          ("Client Id", clientId),
                          ("Grant Type", grantType),
                          ("Authorization Code", authorizationCode),
                          ("Redirect Uri", redirectUri),
                          ("Scope", scope),
                          ("Access Token", accessToken),
                          ("Access Token Expiry", accessTokenExpiry),
                          ("Refresh Token", refreshToken),
                          ("Refresh Token Expiry", refreshTokenExpiry),
                          ("Token Type", tokenType),
                          ("Id Token", idToken)]
        # ("File Data", "file_data")

        # a filter is a lambda
        # that takes parameter x which is a pDict
        # that when evaluated to True will cause
        # the display class to output the packet

        # filterToken is a predicate lambda that evaluates to True
        # only for packets that are responses containing tokens
        # or are requests for tokens
        self.filterToken = lambda x : self.index(x, refreshToken) or self.index(x, grantType) or self.index(x, accessToken) or self.index(x, idToken)

        # filterNone performs no filtering
        # and prints all (http) packets
        self.filterNone = lambda x : True

        # sets the filter to use
        # printAll switches the filter to print all (http) packets
        # the default is the token filter
        if self.printAll:
            self.filterCurrent = self.filterNone
        else:
            self.filterCurrent = self.filterToken

    def index(self, iDict, iKey):
        """
        Determines if iDict contains an entry for key iKey

        Parameters:

        dict iDict - a python dictionary
        str iKey - the key to index into iDict

        Returns True if dict iDict contains a value for key iKey
        Returns False on KeyError
        """
        try:
            iDict[iKey]
        except KeyError:
            return False
        return True

    def outPrint(self, pDict, fieldName, key):
        """
        Output function that prints the entry in pDict
        with key "key" with pretty-printed name "fieldName"

        Parameters:

        dict pDict
        str fieldName
        str key

        Returns:

        data
        """
        try:
            data = pDict[key]
        except KeyError:
            #logger.debug("KeyError on key {0}".format(key))
            return None
        print("{0:21}{1}".format(fieldName + ":", data))
        return data

    def prettyPrint(self, pDict):
        '''
        Pretty-prints the packet data to stdout

        Tightly coupled to the packetDict object

        Parameters:

        dict pDict - the dictionary representing the flattened packet data

        Returns: None
        '''
        logger.debug('Pretty-printing...')
        # printList is the sequence to print
        for item in self.printList:
            self.outPrint(pDict, item[0], item[1])
        print('')

    def jsonWrite(self, pDict):
        """
        Outputs JSON format to stdout

        Parameters:

        dict pDict - the dictionary representing the flattened packet data

        Returns: None
        """
        logger.debug('Outputting JSON')
        jsonString = json.JSONEncoder().encode(pDict)
        print(jsonString)

    def output(self, pDict):
        '''
        Main logging function

        Decides which format to print to stdout

        Parameters: None

        Returns: None
        '''
        logger.debug('Preparing to output...')
        packetTest = self.filterCurrent

        # if packetTest evaluates to True over the pDict
        # then output the packet
        if packetTest(pDict):
            if self.json:
                self.jsonWrite(pDict)
            else:
                self.prettyPrint(pDict)


#!/usr/bin/python3
import matplotlib
matplotlib.use('Agg')
import os
import sys
import pylab

import haitai.graph
import haitai.common
import haitai

if __name__ == '__main__':
    ndays=3000
    bk='全部'
    bk='中小'
    bk='主板'

    # dates
    dates=haitai.common.recent_n_days(ndays)

    # stocks
    th = set(haitai.get_symbol_list(bk))

    d = 'data/163_daily'
    fs = os.listdir(d)
    fs = [f for f in fs if f != '000300.ss']
    print(fs)
    #th = set(list(th)[:100]) # for debug only

    # get data
    pss, vols = haitai.common.load_stock_set(fs,ndays,dates)

    # compute
    p=sum(pss)
    print(p)
    p=p/p[0]
    v=haitai.common.mean_with_nan(vols)

    # fig
    pylab.figure(figsize=(12,6))
    both_ticks=haitai.graph.gen_ticks(dates)

    ax=pylab.subplot(2,1,1)
    haitai.graph.draw_lines(ax,[[p]],log=True)
    haitai.graph.draw_grid(ax,both_ticks)

    ax=pylab.subplot(2,1,2)
    haitai.graph.draw_lines(ax,[[v,'k']],log=True)
    haitai.graph.draw_grid(ax,both_ticks)

    filename=haitai.output_dir+'/'+bk+'_mean.svg'
    pylab.savefig(filename)
    pylab.clf()

    # save mean curve
    file=open(bk+'_mean.txt','w')
    for dd,pp,vv in zip(dates,p,v):
        print(dd,pp,vv,file=file)
    file.close()


from utils import *
import sys

MAX_ITERS = 10000
NUM_FULL_BLOCKS = int(sys.argv[1])
LOW_DEMAND_GAS = int(sys.argv[2])

# the initial basefee is going to be 1.125 ^ i * constants['basefee']
initial_basefee = eip1559_constants["INITIAL_BASEFEE"] * 1.125**NUM_FULL_BLOCKS
print("High congestion basefee", initial_basefee)

# calculates the number of blocks required to get back to the initial gas consumption
def pullback():
    for k in range(0, MAX_ITERS):
        # generate a low consumption dist
        demand = repeat(LOW_DEMAND_GAS, k)

        # the delta is the diff from the current value
        deltas = get_delta(demand)
        basefee = initial_basefee

        for delta in deltas:
            new_basefee = calculate_basefee(basefee, delta)
            print(new_basefee)
            if new_basefee < eip1559_constants["INITIAL_BASEFEE"]:
                print(
                    f"It took {k} blocks of {LOW_DEMAND_GAS} gas for the basefee to get within 1e9 of its original value: new_basefee: {new_basefee}")
                return k
            # update to the new value
            basefee = new_basefee

k = pullback()


#!/usr/bin/env python3

# Process output of pytest - sort log lines by datetime within pytest sections, preserving order of lines with same datetime.

import sys
import re

LOG_RE = re.compile(r'^(\d{2}:\d{2}.\d{3}) ')
PYTEST_SECTION_RE = re.compile(r'^\-+ Captured log \w+ \-+')
SEPARATOR_RE = re.compile('^(\d{2}:\d{2}.\d{3}) [^-]+(\-+ [\w ]+ \-+)')

def main():
    key2lines = {}

    def flush():
        for key in sorted(key2lines):
            for line in key2lines[key]:
                print(line, end='')
        key2lines.clear()

    def push_line(key, line):
        lines = key2lines.setdefault(mo.group(1), [])
        lines.append(line)

    for line in sys.stdin:
        mo = SEPARATOR_RE.match(line)
        if mo:
            push_line(mo.group(1), mo.group(2) + '\n')
            continue
        mo = LOG_RE.match(line)
        if mo:
            push_line(mo.group(1), line)
        else:
            if PYTEST_SECTION_RE.match(line):
                continue
            flush()
            print(line, end='')

main()


# coding:utf-8
""""
Program: coucou
Description:
Author: XY - mailyanxin@gmail.com
Date: 2018-05-31 06:29:05
Last modified: 2018-05-31 06:29:05
Python release: 3.5.2
"""

from decimal import Decimal

from flask import current_app
from flask import jsonify
from flask import request
from flask.views import View
from flask.wrappers import Response
from flask_login import current_user

from coucou.extensions import db
from coucou.utils.helpers import decimal_2_float
from coucou.utils.helpers import set_float_prec

class MyPostView(View):
    '''
    输入参数预处理
    结果自动处理
    '''
    methods = ['POST', ]

    def __init__(self):
        data = request.get_json()
        self.params = self.preprocess_data(data)

        self.logger = current_app.logger
        self.debug = True

        if self.debug:
            self.logger.debug(self.params)

        self.token_name_id = self.params['token_name_id'] if 'token_name_id' in self.params else None
        if self.token_name_id == 'dacc_token':
            self.token_name_id = 'dacc'

        self.token_name = self.params['token_name'] if 'token_name' in self.params else None
        if self.token_name == 'dacc_token':
            self.token_name = 'dacc'
            self.params['token_name'] = 'dacc'

        self.phone_num = current_user.phone_num if not current_user.is_anonymous else None

        for k, v in self.params.items():
            setattr(self, k, v)

        super().__init__()

    def dispatch_request(self):

        self.ret_data = self.logic_view()

        # customize errcode
        if isinstance(self.ret_data, dict) and self.ret_data.get('errcode', 0) != 0:
            return jsonify(self.ret_data)
        # remark
        if isinstance(self.ret_data, dict) and\
            self.ret_data.get('errcode',1) == 0 and\
            self.ret_data.get('remark',None):
            return jsonify(self.ret_data)
        # None
        if self.ret_data is None:
            ret = dict(errcode=0, errmsg='ok')
            if self.debug:
                self.logger.debug(ret)
            return jsonify(ret)
        # 0 errcode
        if isinstance(self.ret_data, dict) or \
                isinstance(self.ret_data, list):
            ret = dict(errcode=0, errmsg='ok')
            self.after_process_data()
            ret['data'] = self.ret_data
            if self.debug:
                self.logger.debug(ret)
            return jsonify(ret)
        # login required ..
        if isinstance(self.ret_data, Response):
            return self.ret_data

        else:
            self.logger.info(self.ret_data)
            self.logger.info(type(self.ret_data))
            return self.ret_data

    def preprocess_data(self, data):

        column_type_info = dict()
        for table_name, table_obj in db.metadata.tables.items():
            colums = table_obj.columns.items()
            for i in colums:
                column_type_info[i[0]] = i[1].type

        new_data = dict()
        for k, v in data.items():
            k_type = column_type_info.get(k, None)
            if k_type:
                if isinstance(k_type, db.String):
                    v = str(v)
                if isinstance(k_type, db.Integer):
                    v = int(v)
                if isinstance(k_type, db.Float):
                    v = float(v)
                if isinstance(k_type, db.DECIMAL):
                    if isinstance(v, float):
                        v = Decimal(str(v))
                    else:
                        v = Decimal(v)

            new_data[k] = v

        return new_data

    def after_process_data(self):

        if isinstance(self.ret_data, dict):
            new_data = dict()
            for k, v in self.ret_data.items():
                if isinstance(v, Decimal):
                    new_data[k] = decimal_2_float(v)
                if isinstance(v, float):
                    if 'star' in k:
                        new_data[k] = set_float_prec(v)
                    else:
                        new_data[k] = v
                else:
                    new_data[k] = v

            self.ret_data = new_data
        else:
            pass


"""Sort array to remove duplicates."""

def sort(l):
    """Return sorted array without duplicates."""
    less = []
    equal = []
    greater = []
    if len(l) > 1:
        pivot = l[0]
        for x in l:
            if x < pivot:
                less.append(x)
            if x == pivot and len(equal) < 1:
                equal.append(x)
            if x > pivot:
                greater.append(x)
        return sort(less) + equal + sort(greater)
    else:
        return l

def main():
    """Execute main body."""
    base = [5, 5, 5, 5, 56, 6, 4, 223, 45, 342, 4, 53, 423, 4, 4, 3, 2, 1]
    copy = base[:]
    print("Base {} \nis now {}.".format(copy, sort(base)))

if __name__ == "__main__":
    """Execute main() on open."""
    main()


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.svm import SVC
from sklearn.linear_model import SGDRegressor
from sklearn.metrics import roc_auc_score, r2_score

# import the data file
df = pd.read_csv('train_features.csv')
dy = pd.read_csv('train_labels.csv')
dt = pd.read_csv('test_features.csv')

#---Compare the number of missing values in each column---
#missing = df.isnull().sum(0).reset_index()
#missing.columns = ['column', 'count']
#missing = missing.sort_values(by = 'count', ascending = False).loc[missing['count'] > 0]
#missing['percentage'] = missing['count'] / float(df.shape[0]) * 100
#ind = np.arange(missing.shape[0])
#width = 1
#fig, ax = plt.subplots(figsize=(8,10))
#rects = ax.barh(ind, missing.percentage.values, color='b')
#ax.set_yticks(ind)
#ax.set_yticklabels(missing.column.values, rotation='horizontal')
#ax.set_xlabel("Precentage of missing values %", fontsize = 10)
#ax.set_title("Number of missing values in each column", fontsize = 12)
#plt.show()
#--------------------------------------------------------

#-----------------Correlation diagram--------------------
#colormap = plt.cm.RdBu
#plt.figure(figsize=(24,22))
#plt.title('Pearson Correlation of Features', y=1.05, size=15)
#sns.heatmap(X_trains.astype(float).corr(),linewidths=0.1, vmax=1.0, \
#            square=True, cmap=colormap, linecolor='white', annot=True)
#--------------------------------------------------------

# assign X and y from the train dataset
all_pids = df.groupby('pid')
ap = list(all_pids.groups.keys())
train_pids, val_pids = train_test_split(ap, test_size = 0.2, \
                                        random_state = 42)

vital = ['Heartrate', 'SpO2', 'ABPs', 'ABPm', 'RRate','ABPd']
test = ['Temp', 'EtCO2', 'PTT', 'BUN', 'Lactate', 'Hgb', 'HCO3', 'BaseExcess', \
         'Fibrinogen', 'Phosphate', 'WBC', 'Creatinine', 'PaCO2', 'AST', \
         'FiO2', 'Platelets', 'SaO2', 'Glucose', 'Magnesium', 'Potassium', \
         'Calcium', 'Alkalinephos', 'Bilirubin_direct', 'Chloride', \
         'Hct', 'Bilirubin_total', 'TroponinI', 'pH']
tl = ['LABEL_BaseExcess', 'LABEL_Fibrinogen', 'LABEL_AST', 'LABEL_Alkalinephos', \
      'LABEL_Bilirubin_total', 'LABEL_Lactate', 'LABEL_TroponinI', 'LABEL_SaO2', \
      'LABEL_Bilirubin_direct', 'LABEL_EtCO2', 'LABEL_Sepsis']
vl = ['LABEL_RRate', 'LABEL_ABPm', 'LABEL_SpO2', 'LABEL_Heartrate']

#X_train = pd.DataFrame()
#X_val = pd.DataFrame()
#Y_train = pd.DataFrame()
#Y_val = pd.DataFrame()
#itt = IterativeImputer(max_iter=20, tol=0.01, random_state = 42)
#preprocessor = ColumnTransformer(transformers = [ \
#                                        ('vital', itt, vital)])
#sc = StandardScaler()
#
##Imputing missing values
#X_i = df.copy()
#X_i.loc[:,test] = X_i.loc[:,test].notnull().astype('float')
#imp = preprocessor.fit(X_i)
#X_i.loc[:,vital] = imp.transform(X_i)
#X_i = X_i.drop(['Bilirubin_total', 'PaCO2', 'BaseExcess', 'Alkalinephos', \
#             'HCO3', 'PTT', 'Phosphate', 'Magnesium', 'Creatinine', \
#             'Calcium', 'Platelets', 'WBC'], axis = 1)
#
#for pid in train_pids:
#    X_pid = X_i.groupby('pid').get_group(pid)
#    X_pid = X_pid.drop(['pid', 'Time'], axis=1)
#    xp = X_pid.iloc[11].values
#    xi = X_pid.iloc[:,:].drop(['Age'], axis=1).values.flatten()
#    xj = pd.DataFrame([[*xi, *xp]])
#    X_train = X_train.append(xj)
#X_trains = pd.DataFrame(sc.fit_transform(X_train))
#
#for pid in val_pids:
#    X_pid = X_i.groupby('pid').get_group(pid)
#    X_pid = X_pid.drop(['pid', 'Time'], axis=1)
#    xp = X_pid.iloc[11].values
#    xi = X_pid.iloc[:,:].drop(['Age'], axis=1).values.flatten()
#    xj = pd.DataFrame([[*xi, *xp]])
#    X_val = X_val.append(xj)
#X_vals = pd.DataFrame(sc.fit_transform(X_val))
#
#
##Split y values into train and validation set
#ypids = dy.groupby('pid')
#for pid in train_pids:
#    Y_pid = ypids.get_group(pid)
#    Y_pid = Y_pid.drop(['pid'], axis=1)
#    Y_train = Y_train.append(Y_pid)
#
#for pid in val_pids:
#    Y_pid = ypids.get_group(pid)
#    Y_pid = Y_pid.drop(['pid'], axis=1)
#    Y_val = Y_val.append(Y_pid)
#
##Test set
#X_t = dt.copy()
#X_t.loc[:,test] = X_t.loc[:,test].notnull().astype('float')
#X_t.loc[:,vital] = imp.transform(X_t)
#X_t = X_t.drop(['Bilirubin_total', 'PaCO2', 'BaseExcess', 'Alkalinephos', \
#             'HCO3', 'PTT', 'Phosphate', 'Magnesium', 'Creatinine', \
#             'Calcium', 'Platelets', 'WBC'], axis = 1)
#
#test_pid = list(dt.groupby('pid').groups.keys())
#X_test = pd.DataFrame()
#for pid in test_pid:
#    X_pid = X_t.groupby('pid').get_group(pid)
#    X_pid = X_pid.drop(['pid', 'Time'], axis=1)
#    xp = X_pid.iloc[11].values
#    xi = X_pid.iloc[:,:].drop(['Age'], axis=1).values.flatten()
#    xj = pd.DataFrame([[*xi, *xp]])
#    X_test = X_test.append(xj)
#X_tests = pd.DataFrame(sc.fit_transform(X_test))

svc = SVC(kernel='rbf', C=0.1, class_weight='balanced', gamma='scale', \
          probability=True, random_state=0)
svr = SGDRegressor(penalty='elasticnet', alpha=0.05, l1_ratio=0.1)
Y_predd = pd.DataFrame()
Y_predd = Y_predd.append([pd.DataFrame(test_pid, columns=['pid'])])

#for i in ['LABEL_Sepsis']:
#    svc.fit(X_trains, Y_train[i])
#    y_pred = svc.predict(X_vals)
#    print('roc_auc_score:', roc_auc_score(Y_val[i], y_pred))

for i in tl:
    svc.fit(X_trains, Y_train[i])
    y_pred = svc.predict_proba(X_tests)
    Y_predd[i] = pd.DataFrame(y_pred)[1]

#for i in vl:
#    svr.fit(X_trains, Y_train[i])
#    y_pred = svr.predict(X_vals)
#    print('r2 score', r2_score(Y_val[i], y_pred))

for i in vl:
    svr.fit(X_trains, Y_train[i])
    y_pred = svr.predict(X_tests)
    Y_predd[i] = pd.DataFrame(y_pred)

Y_predd.to_csv('prediction.zip', index=False, float_format='%.3f', compression='zip')


from angular_flask import app
import influxdb
from influxdb import InfluxDBClient

class ConnectInflux():
	def __init__(self):
		self.client = InfluxDBClient('influxdb-prod.juspay.in',8086,'realtime-user', 'MgqEK,nWx2bV','godel-realtime')

	def acsMetrics(self):
		query1 = "select count(distinct(session_id)) as sess_count from \
				payment_details_data where time > now() - 5m and session_id<>'null' and \
				bank<>'null' and app_name<>'null' group by \
				acs_hash"
		query2 = "select count(distinct(session_id)) from \
				payment_details_data where time > now() - 5m and session_id<>'null' and \
				bank<>'null' and app_name<>'null' and payment_status='SUCCESS' group by \
				acs_hash"
		query3 = "select count(distinct(session_id)) ,sum(potential_payment_flow_error) \
				  as pay_error,sum(user_error) as user_error ,sum(godel_exception) as exp,\
				  sum(acs_error) as acs_error from session_aggregates where time> now() - 5m \
				  and session_id<>'null' and bank<>'null' and app_name<>'null' group by acs_hash"
		result1 = self.client.query(query1)
		result2 = self.client.query(query2)
		result3 = self.client.query(query3)

		join_list = {}
		for i in result1[0]['points']:
			join_list[i[2]] = [i[1],0,0,0,0,0]  #l-4 :u'pay_error', u'user_error', u'exp', u'acs_error', u'acs_hash'

		for i in result2[0]['points']:
			if i[2] in join_list:
				join_list[i[2]][1] = i[1]
			else:
				join_list[i[2]] = [0,i[1],0,0,0,0]

		for i in result3[0]['points']:
			if i[6] in join_list:
				join_list[i[6]][2] = i[2]
				join_list[i[6]][3] = i[3]
				join_list[i[6]][4] = i[4]
				join_list[i[6]][5] = i[5]
			else:
				join_list[i[6]] = [0,0,i[2],i[3],i[4],i[5]]

		res = {'columns':['acs_hash','tot_sess_count','success_count','success_rate','potential_error', 'user_error', 'godel_exception', 'acs_error'],'rows':[]}
		map_column = {}
		for i in join_list:
			row = []
			row.append(i)
			row.append(join_list[i][0])
			row.append(join_list[i][1])
			if join_list[i][0]==0:
				row.append('not-def')
			else:
				row.append(round(float(join_list[i][1])/float(join_list[i][0])*100,2))
			row.append(join_list[i][2])
			row.append(join_list[i][3])
			row.append(join_list[i][4])
			row.append(join_list[i][5])
			res['rows'].append(row)

		return res

	def liveSessionStream(self):
		query = "select session_id, acs_hash, bank, name, numscreens, numevents, acs_error,\
				 potential_payment_flow_error,user_error,godel_exception from \
				 internal_session_aggregates where time > now() - 5m"

		res = {'columns':[],'rows':[]}
		result = self.client.query(query)
		columns=['session_id','acs_hash','bank','name' ,'numscreens','acs_error', 'potential_payment_flow_error','user_error','godel_exception']
		map_list = {}

		for i,j in enumerate(result[0]['columns']):
			map_list[j] =i

		for i in result[0]['points']:
			arr = []
			for j in columns:
				arr.append(i[map_list[j]])
			res['rows'].append(arr)

		res['columns'] = columns
		return res


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /tmp/pip-build-ed191__6/requests-toolbelt/requests_toolbelt/auth/http_proxy_digest.py
# Compiled at: 2020-01-10 16:25:32
# Size of source mod 2**32: 3705 bytes
"""The module containing HTTPProxyDigestAuth."""
import re
from requests import cookies, utils
from . import _digest_auth_compat as auth

class HTTPProxyDigestAuth(auth.HTTPDigestAuth):
    __doc__ = 'HTTP digest authentication between proxy\n\n    :param stale_rejects: The number of rejects indicate that:\n        the client may wish to simply retry the request\n        with a new encrypted response, without reprompting the user for a\n        new username and password. i.e., retry build_digest_header\n    :type stale_rejects: int\n    '
    _pat = re.compile('digest ', flags=(re.IGNORECASE))

    def __init__(self, *args, **kwargs):
        (super(HTTPProxyDigestAuth, self).__init__)(*args, **kwargs)
        self.stale_rejects = 0
        self.init_per_thread_state()

    @property
    def stale_rejects(self):
        thread_local = getattr(self, '_thread_local', None)
        if thread_local is None:
            return self._stale_rejects
        else:
            return thread_local.stale_rejects

    @stale_rejects.setter
    def stale_rejects(self, value):
        thread_local = getattr(self, '_thread_local', None)
        if thread_local is None:
            self._stale_rejects = value
        else:
            thread_local.stale_rejects = value

    def init_per_thread_state(self):
        try:
            super(HTTPProxyDigestAuth, self).init_per_thread_state()
        except AttributeError:
            pass

    def handle_407(self, r, **kwargs):
        """Handle HTTP 407 only once, otherwise give up

        :param r: current response
        :returns: responses, along with the new response
        """
        if r.status_code == 407 and self.stale_rejects < 2:
            s_auth = r.headers.get('proxy-authenticate')
            if s_auth is None:
                raise IOError('proxy server violated RFC 7235:407 response MUST contain header proxy-authenticate')
            else:
                if not self._pat.match(s_auth):
                    return r
            self.chal = utils.parse_dict_header(self._pat.sub('', s_auth, count=1))
            if 'Proxy-Authorization' in r.request.headers:
                if 'stale' in self.chal:
                    if self.chal['stale'].lower() == 'true':
                        self.stale_rejects += 1
                    elif self.chal['stale'].lower() == 'false':
                        raise IOError('User or password is invalid')
            r.content
            r.close()
            prep = r.request.copy()
            cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)
            prep.headers['Proxy-Authorization'] = self.build_digest_header(prep.method, prep.url)
            _r = (r.connection.send)(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep
            return _r
        else:
            return r

    def __call__(self, r):
        self.init_per_thread_state()
        if self.last_nonce:
            r.headers['Proxy-Authorization'] = self.build_digest_header(r.method, r.url)
        r.register_hook('response', self.handle_407)
        return r


# Definition for a undirected graph node
class UndirectedGraphNode:

    def __init__(self, x):
        self.label = x
        self.neighbors = []

class Solution:
    # @param node, a undirected graph node
    # @return a undirected graph node

    def cloneGraph(self, orig_root):
        if not orig_root:
            return None
        if not orig_root.neighbors:
            a = UndirectedGraphNode(orig_root.label)
            return a

        vertex_map = {}
        # create new node from given input
        copy_root = UndirectedGraphNode(orig_root.label)

        q = []
        q.append(orig_root)  # add the original input node to the Q

        # map from original root to new clone graph's node
        vertex_map[orig_root] = copy_root

        # perform BFS
        while len(q) >= 1:
            u = q.pop()

            for v in u.neighbors:
                if v not in vertex_map:
                    # Create a new node and map it to v
                    vertex_map[v] = UndirectedGraphNode(v.label)
                    q.append(v)

                # below line translate into copy_of_u.neighbors.append(v) . We
                # are not actually returning the vertex map
                # the outputs of the map are used to link the nodes together!

                # below we cannot do append(v) as that would point back to the original nodes
                # we have to do append(vertex_map[v]) as that would point to
                # the copy
                vertex_map[u].neighbors.append(vertex_map[v])

                # i.e add this node as a neighbor of the copy node
                # vertex_map[u] = copy_of_u

        # -> this gives the root of the graph clone
        return vertex_map[orig_root]


# -*- coding: utf-8 -*-
import os
import re
import tempfile
from test.data import (
    bob,
    cheese,
    context0,
    context1,
    context2,
    hates,
    likes,
    michel,
    pizza,
    tarek,
)

import pytest
from rdflib import RDF, RDFS, BNode, Literal, URIRef, Variable
from rdflib.graph import Dataset, QuotedGraph

storename = "SQLiteStore"
path = tempfile.mktemp(prefix="test", dir="/tmp")

graphuri = URIRef("urn:example:graph")
othergraphuri = URIRef("urn:example:othergraph")

implies = URIRef("http://www.w3.org/2000/10/swap/log#implies")

test_n3 = """\
@prefix rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://test/> .
{:a :b :c;a :foo} => {:a :d :c,?y} .
_:foo a rdfs:Class .
:a :d :c .
"""

def expunge_path():
    if path is not None:
        if os.path.exists(path):
            if os.path.isdir(path):
                for f in os.listdir(path):
                    os.unlink(path + "/" + f)
                os.rmdir(path)
            elif len(path.split(":")) == 1:
                os.unlink(path)
            else:
                os.remove(path)

@pytest.fixture(scope="function")
def dataset():

    expunge_path()
    graph = Dataset(store=storename, identifier=context0)
    graph.open(path, create=True)

    yield graph

    graph.close()

    expunge_path()

def test_simple_graph(dataset):
    graph = dataset.get_context(graphuri)
    graph.add((tarek, likes, pizza))
    graph.add((bob, likes, pizza))
    graph.add((bob, likes, cheese))

    g2 = dataset.get_context(othergraphuri)
    g2.add((michel, likes, pizza))

    assert len(graph) == 3, "graph contains 3 triples"
    assert len(g2) == 1, "other graph contains 1 triple"

    r = graph.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")
    assert len(list(r)) == 2, "two people like pizza"

    r = graph.triples((None, likes, pizza))
    assert len(list(r)) == 2, "two people like pizza"

    # Test initBindings
    r = graph.query(
        "SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }",
        initBindings={"s": tarek},
    )
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = graph.triples((tarek, likes, pizza))
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = graph.triples((tarek, likes, cheese))
    assert len(list(r)) == 0, "tarek doesn't like cheese"

    g2.add((tarek, likes, pizza))
    graph.remove((tarek, likes, pizza))
    r = graph.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")

def test_dataset_default(dataset):
    dataset.default_union = True

    subgraph1 = dataset.get_context(context1)

    subgraph1.add((tarek, likes, pizza))

    subgraph2 = dataset.get_context(context2)
    subgraph2.add((bob, likes, pizza))

    subgraph1.add((tarek, hates, cheese))

    assert len(subgraph1) == 2, "graph contains 2 triples"

    # the following are actually bad tests as they depend on your endpoint,
    # as pointed out in the sparqlstore.py code:
    #
    # # For Datasets, reading is done from the "default graph" Exactly
    # # what this means depends on your endpoint, because SPARQL does not offer a
    # # simple way to query the union of all graphs as it would be expected for a
    # # ConjuntiveGraph.
    # #
    # # Fuseki/TDB has a flag for specifying that the default graph
    # # is the union of all graphs (tdb:unionDefaultGraph in the Fuseki config).
    assert (
        len(dataset) == 3
    ), f"default union graph should contain three triples but contains {list(dataset)}:\n"

    r = dataset.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")

    assert len(list(r)) == 2, f"two people should like pizza, not {len(list(r))}"

    r = dataset.query(
        "SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }",
        initBindings={"s": tarek},
    )
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = dataset.triples((tarek, likes, pizza))
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = dataset.triples((tarek, likes, cheese))
    assert len(list(r)) == 0, "tarek doesn't like cheese"

    subgraph2.remove((bob, likes, pizza))

    r = dataset.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")
    assert len(list(r)) == 1, "only tarek likes pizza"

def test_update(dataset):
    dataset.update(
        "INSERT DATA { GRAPH <urn:example:graph> { <urn:example:michel> <urn:example:likes> <urn:example:pizza> . } }"
    )

    graph = dataset.get_context(graphuri)
    assert len(graph) == 1, "graph contains 1 triple"

def test_update_with_initns(dataset):
    dataset.update(
        "INSERT DATA { GRAPH ns:graph { ns:michel ns:likes ns:pizza . } }",
        initNs={"ns": URIRef("urn:example:")},
    )

    graph = dataset.get_context(graphuri)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

def test_update_with_initbindings(dataset):
    dataset.update(
        "INSERT { GRAPH <urn:example:graph> { ?a ?b ?c . } } WherE { }",
        initBindings={
            "a": michel,
            "b": likes,
            "c": pizza,
        },
    )

    graph = dataset.get_context(graphuri)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

def test_multiple_update_with_initbindings(dataset):
    dataset.update(
        "INSERT { GRAPH <urn:example:graph> { ?a ?b ?c . } } WHERE { };"
        "INSERT { GRAPH <urn:example:graph> { ?d ?b ?c . } } WHERE { }",
        initBindings={
            "a": michel,
            "b": likes,
            "c": pizza,
            "d": bob,
        },
    )

    graph = dataset.get_context(graphuri)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza), (bob, likes, pizza)]
    ), "michel and bob like pizza"

def test_named_graph_r3_update(dataset):
    graph = dataset.get_context(graphuri)

    # Strings with unbalanced curly braces
    tricky_strs = ["With an unbalanced curly brace %s " % brace for brace in ["{", "}"]]
    for tricky_str in tricky_strs:
        r3 = (
            """INSERT { ?b <urn:example:says> "%s" }
        WHERE { ?b <urn:example:likes> <urn:example:pizza>} """
            % tricky_str
        )
        graph.update(r3)

def test_named_graph_update(dataset):
    graph = dataset.get_context(graphuri)
    r1 = "INSERT DATA { <urn:example:michel> <urn:example:likes> <urn:example:pizza> }"
    graph.update(r1)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

    r2 = (
        "DELETE { <urn:example:michel> <urn:example:likes> <urn:example:pizza> } "
        + "INSERT { <urn:example:bob> <urn:example:likes> <urn:example:pizza> } WHERE {}"
    )
    graph.update(r2)
    assert set(graph.triples((None, None, None))) == set(
        [(bob, likes, pizza)]
    ), "only bob likes pizza"
    says = URIRef("urn:example:says")

    # Strings with unbalanced curly braces
    tricky_strs = ["With an unbalanced curly brace %s " % brace for brace in ["{", "}"]]
    for tricky_str in tricky_strs:
        r3 = (
            """INSERT { ?b <urn:example:says> "%s" }
        WHERE { ?b <urn:example:likes> <urn:example:pizza>} """
            % tricky_str
        )
        graph.update(r3)

    values = set()
    for v in graph.objects(bob, says):
        values.add(str(v))
    assert values == set(tricky_strs)

# @pytest.mark.xfail(reason="Failure in SQL query string handling")
def test_named_graph_update_complicated_strings(dataset):
    graph = dataset.get_context(graphuri)
    says = URIRef("urn:example:says")
    # Complicated Strings
    r4strings = []
    r4strings.append(r'''"1: adfk { ' \\\" \" { "''')
    r4strings.append(r'''"2: adfk } <foo> #éï \\"''')

    r4strings.append(r"""'3: adfk { " \\\' \' { '""")
    r4strings.append(r"""'4: adfk } <foo> #éï \\'""")

    r4strings.append(r'''"""5: adfk { ' \\\" \" { """''')
    r4strings.append(r'''"""6: adfk } <foo> #éï \\"""''')
    r4strings.append('"""7: ad adsfj \n { \n sadfj"""')

    r4strings.append(r"""'''8: adfk { " \\\' \' { '''""")
    r4strings.append(r"""'''9: adfk } <foo> #éï \\'''""")
    r4strings.append("'''10: ad adsfj \n { \n sadfj'''")

    r4 = "\n".join(
        [
            "INSERT DATA { <urn:example:michel> <urn:example:says> %s } ;" % s
            for s in r4strings
        ]
    )
    graph.update(r4)
    values = set()
    for v in sorted(graph.objects(michel, says)):
        values.add(str(v))
    assert values == set(
        [
            re.sub(
                r"\\(.)",
                r"\1",
                re.sub(r"^'''|'''$|^'|'$|" + r'^"""|"""$|^"|"$', r"", s),
            )
            for s in r4strings
        ]
    )

# @pytest.mark.xfail(reason="Failure in SQL query string handling")
def test_named_graph_update_iri_containing_octothorpe(dataset):
    graph = dataset.get_context(graphuri)

    # IRI Containing ' or #
    # The fragment identifier must not be misinterpreted as a comment
    # (commenting out the end of the block).
    # The ' must not be interpreted as the start of a string, causing the }
    # in the literal to be identified as the end of the block.
    r5 = """INSERT DATA { <urn:example:michel> <urn:example:hates> <urn:example:foo'bar?baz;a=1&b=2#fragment>, "'}" }"""

    graph.update(r5)
    values = set()
    for v in graph.objects(michel, hates):
        values.add(str(v))
    assert values == set(["urn:example:foo'bar?baz;a=1&b=2#fragment", "'}"])

    # Comments
    r6 = """
        INSERT DATA {
            <urn:example:bob> <urn:example:hates> <urn:example:bob> . # No closing brace: }
            <urn:example:bob> <urn:example:hates> <urn:example:michel>.
        }
    #Final { } comment"""

    graph.update(r6)
    values = set()
    for v in graph.objects(bob, hates):
        values.add(v)
    assert values == set([bob, michel])

def test_named_graph_update_with_initbindings(dataset):
    graph = dataset.get_context(graphuri)
    r = "INSERT { ?a ?b ?c } WHERE {}"
    graph.update(r, initBindings={"a": michel, "b": likes, "c": pizza})
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

def test_empty_literal(dataset):
    # test for https://github.com/RDFLib/rdflib/issues/457
    # also see test_issue457.py which is sparql store independent!
    graph = dataset.get_context(graphuri)
    graph.add(
        (
            URIRef("http://example.com/s"),
            URIRef("http://example.com/p"),
            Literal(""),
        )
    )

    o = tuple(graph)[0][2]
    assert Literal("") == o, repr(o)

def test_n3_store(dataset):
    dataset.default_union = True
    dataset.parse(data=test_n3, format="n3")
    formula_a = BNode()
    formula_b = BNode()
    for s, p, o in dataset.triples((None, implies, None)):
        formula_a = s
        formula_b = o

    assert type(formula_a) == QuotedGraph and type(formula_b) == QuotedGraph
    a = URIRef("http://test/a")
    b = URIRef("http://test/b")
    c = URIRef("http://test/c")
    d = URIRef("http://test/d")
    v = Variable("y")

    universe = Dataset(dataset.store)

    # test formula as terms
    assert len(list(universe.triples((formula_a, implies, formula_b)))) == 1

    # test variable as term and variable roundtrip
    assert len(list(formula_b.triples((None, None, v)))) == 1
    for s, p, o in formula_b.triples((None, d, None)):
        if o != c:
            assert isinstance(o, Variable)
            assert o == v
    s = list(universe.subjects(RDF.type, RDFS.Class))[0]
    assert isinstance(s, BNode)
    assert len(list(universe.triples((None, implies, None)))) == 1
    assert len(list(universe.triples((None, RDF.type, None)))) == 1
    assert len(list(formula_a.triples((None, RDF.type, None)))) == 1
    assert len(list(formula_a.triples((None, None, None)))) == 2
    assert len(list(formula_b.triples((None, None, None)))) == 2
    assert len(list(universe.triples((None, None, None)))) == 3
    assert len(list(formula_b.triples((None, URIRef("http://test/d"), None)))) == 2
    assert len(list(universe.triples((None, URIRef("http://test/d"), None)))) == 1

    # context tests
    # test contexts with triple argument
    assert len(list(universe.contexts((a, d, c)))) == 1

    # Remove test cases
    universe.remove((None, implies, None))
    assert len(list(universe.triples((None, implies, None)))) == 0
    assert len(list(formula_a.triples((None, None, None)))) == 2
    assert len(list(formula_b.triples((None, None, None)))) == 2

    formula_a.remove((None, b, None))
    assert len(list(formula_a.triples((None, None, None)))) == 1
    formula_a.remove((None, RDF.type, None))
    assert len(list(formula_a.triples((None, None, None)))) == 0

    universe.remove((None, RDF.type, RDFS.Class))

    # remove_context tests
    universe.remove_graph(formula_b)
    assert len(list(universe.triples((None, RDF.type, None)))) == 0
    assert len(universe) == 1
    assert len(formula_b) == 0

    universe.remove((None, None, None))
    assert len(universe) == 0

    dataset.close()
    dataset.store.destroy(path)

xmltestdoc = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
   xmlns="http://example.org/"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
>
  <rdf:Description rdf:about="http://example.org/a">
    <b rdf:resource="http://example.org/c"/>
  </rdf:Description>¬
</rdf:RDF>
"""

n3testdoc = """@prefix : <http://example.org/> .

:a :b :c .
"""

nttestdoc = "<http://example.org/a> <http://example.org/b> <http://example.org/c> .\n"


from microprediction.config_private import DODDLE_MAMMAL
from microprediction import MicroWriter
import time
from pprint import pprint
import numpy as np

# This example illustrates how easy it is to enter z1~ streams
# For a video explanation of z1 streams and see https://www.linkedin.com/feed/update/urn:li:activity:6696569137365430272/
# See https://www.linkedin.com/pulse/short-introduction-z-streams-peter-cotton-phd/ for an explanation of z-streams.

# You need to set a write key.
# Supplying a URL to your repository is optional.
# This will do is make your position on the leaderboards clickable and direct people to your work.
# You can alternatively provide your personal homepage or LinkedIn profile

WRITE_KEY = DODDLE_MAMMAL
ANIMAL = MicroWriter.animal_from_key(DODDLE_MAMMAL)
REPO = 'https://github.com/microprediction/microprediction/blob/master/crawler_alternatives/' + ANIMAL.lower().replace(
    ' ', '_') + '.py'

# We simply loop over all z1~ streams
# These streams are approximately normally distributed.
# We enter N(0,1) distributed samples so as to try to take advantage of algorithms
# which accidentally submit distributional predictions far from this.
# This code could be run as a cron job or scheduled task at PythonAnywhere, for example

def jiggle():
    return 0.001 * np.random.randn()

if __name__ == "__main__":
    mw = MicroWriter(write_key=WRITE_KEY)
    for name in mw.get_streams():
        if 'z1~' in name:
            values = [mw.norminv(p) + jiggle() for p in mw.percentiles()]
            for delay in mw.DELAYS:
                pprint(mw.submit(name=name, values=values, delay=delay))
                time.sleep(7.28) # No need to overwhelm server
                print('Submitted to '+name+' '+str(delay), flush=True)


import mock
import os
import unittest

from ngi_pipeline.engines.sarek.exceptions import ParserException
from ngi_pipeline.engines.sarek.models.sample import SarekAnalysisSample
from ngi_pipeline.engines.sarek.models.workflow import SarekWorkflowStep, SarekMainStep
from ngi_pipeline.tests.engines.sarek.models.test_sarek import TestSarekGermlineAnalysis

class TestSarekWorkflowStep(unittest.TestCase):

    CONFIG = TestSarekGermlineAnalysis.CONFIG

    def setUp(self):
        self.sarek_args = self.CONFIG["sarek"].copy()
        self.sarek_workflow_step = SarekWorkflowStep(
            self.sarek_args["command"],
            **{k: v for k, v in self.sarek_args.items() if k != "command"})

    def test__append_argument(self):
        base_string = "this-is-the-base-string"

        # test a non-existing attribute
        attribute = "this-attribute-does-not-exist"
        self.assertEqual(base_string, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a None attribute
        attribute = "none_attribute"
        setattr(self.sarek_workflow_step, attribute, None)
        self.assertEqual(base_string, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a list attribute
        attribute = "list_attribute"
        value = ["this", "is", "a", "list"]
        self.sarek_workflow_step.parameters[attribute] = value
        expected_result = "{0} --{1} ${{{1}}}".format(base_string, attribute)
        self.assertEqual(expected_result, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a string attribute
        attribute = "string_attribute"
        value = "this-is-a-string"
        self.sarek_workflow_step.parameters[attribute] = value
        expected_result = "{0} --{1} ${{{1}}}".format(base_string, attribute)
        self.assertEqual(expected_result, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a custom hyphen string
        hyphen = "xyz"
        expected_result = "{0} {2}{1} ${{{1}}}".format(base_string, attribute, hyphen)
        self.assertEqual(
            expected_result, self.sarek_workflow_step._append_argument(
                base_string, attribute, hyphen=hyphen))

    def test_sarek_step(self):
        with self.assertRaises(NotImplementedError):
            self.sarek_workflow_step.sarek_step()

    def test_command_line(self):
        sarek_workflow_step = SarekWorkflowStep(self.sarek_args["command"])
        self.assertEqual(self.sarek_args["command"], sarek_workflow_step.command_line())

    def test_command_line_args(self):
        valid_tools = self.sarek_args["tools"]
        observed_command_line = self.sarek_workflow_step.command_line()
        self.assertIn("--tools", observed_command_line)
        self.assertIn(",".join(valid_tools), observed_command_line)
        for key in [x for x in list(self.sarek_args.keys()) if x not in ["tools", "command"]]:
            self.assertIn("-{} {}".format(key, self.sarek_args[key]), observed_command_line)

class TestSarekMainStep(unittest.TestCase):

    def test_report_files(self):
        analysis_sample = mock.Mock(spec=SarekAnalysisSample)
        analysis_sample.sampleid = "this-is-a-sample-id"
        analysis_sample.sample_analysis_path.return_value = "this-is-a-path"
        analysis_sample.sample_analysis_results_dir.return_value = "this-is-the-results-path"
        with mock.patch("os.listdir") as list_mock:
            file_list = ["file1", "file2.extension", "file3.metrics", "file4.metrics"]
            for listed_files in [file_list[0:2], file_list]:
                list_mock.return_value = listed_files
                with self.assertRaises(ParserException):
                    SarekMainStep.report_files(analysis_sample)
            list_mock.return_value = file_list[0:3]
            self.assertEqual(
                file_list[2], os.path.basename(SarekMainStep.report_files(analysis_sample)[1][1]))


# coding: utf-8

# In[ ]:

import datetime
from sklearn.feature_selection import chi2, SelectPercentile
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_extraction.text import CountVectorizer
from scipy import sparse
import lightgbm as lgb
import warnings
import time
import pandas as pd
import numpy as np
import os

# In[ ]:

train = pd.read_hdf('../data/train_data2.hdf', key='xunfei')

test = pd.read_hdf('../data/test_data2.hdf', key='xunfei')
data = pd.concat([train, test], axis=0, ignore_index=True)

# In[ ]:

pd.set_option('display.max_columns',100)
data['time']=(data['timee']-data['timee'].min())/(data['timee'].max()-data['timee'].min())

# In[ ]:

ad_cate_feature=['adid','advert_id','orderid','campaign_id','creative_id','creative_tp_dnf','creative_type',
       'creative_is_jump','creative_is_download','creative_has_deeplink','advert_industry_inner0','advert_industry_inner1']
media_cate_feature=['app_cate_id','f_channel','app_id','inner_slot_id']
content_cate_feature=['city', 'carrier', 'province', 'nnt', 'devtype', 'osv', 'make', 'model']

all_data=data.copy()

need=list(all_data.index)

# 循环读取
import gc
from tqdm import tqdm#'unsparse','timesparse',,'expsparse1','expsparse2','dupsparse','innersparse1','dupsparse1'
filelist=['expsparsex','expsparse1','innersparse1','appearsparses']#'svdsparsenew',
start=1
tail=0
##all_df=sparse.csr_matrix((len(all_data),0))
maxrec={}
for f in filelist:

    ttf=pd.read_hdf( '../data/'+f+'.hdf').astype(np.float32).iloc[need,:]
    orgshape=ttf.shape[1]

    '''use=pd.read_hdf('../data/'+f+'_catvip'+'.hdf',key='xunfei')['catvip'].values

    ttf=ttf.iloc[:,use]
    maxrec[f]=use#ttf.shape[1]'''
    if (start==1):
        all_df=ttf.copy()
        start=0
    else:
        all_df=pd.concat([all_df,ttf],axis=1)
    print(f,orgshape,(ttf.shape[1])/orgshape)
    del ttf
    gc.collect()

# In[ ]:

from tqdm import tqdm
num_feature = ['creative_width', 'creative_height', 'hour','user_tags_len']#,'time'
ttmp=data[num_feature].values.astype(np.int32)[need,:]
for col in tqdm(range(ttmp.shape[1])):
    all_df['num'+str(col)]=ttmp[:,col]

del ttmp
gc.collect()

# In[ ]:

# 默认加载 如果 增加了cate类别特征 请改成false重新生成
import os
if os.path.exists('../feature/base_train_csr.npz') and True:
    print('load_csr---------')
    base_train_csr = sparse.load_npz( '../feature/base_train_csr.npz').tocsr().astype('bool')
    base_predict_csr = sparse.load_npz('../feature/base_predict_csr.npz').tocsr().astype('bool')
else:
    base_train_csr = sparse.csr_matrix((len(train), 0))
    base_predict_csr = sparse.csr_matrix((len(predict_x), 0))

    enc = OneHotEncoder()
    for feature in cate_feature:
        enc.fit(data[feature].values.reshape(-1, 1))
        base_train_csr = sparse.hstack((base_train_csr, enc.transform(train_x[feature].values.reshape(-1, 1))), 'csr',
                                       'bool')
        base_predict_csr = sparse.hstack((base_predict_csr, enc.transform(predict[feature].values.reshape(-1, 1))),
                                         'csr',
                                         'bool')
    print('one-hot prepared !')

    cv = CountVectorizer(min_df=10)
    for feature in ['user_tagss']:
        data[feature] = data[feature].astype(str)
        cv.fit(data[feature])
        base_train_csr = sparse.hstack((base_train_csr, cv.transform(train_x[feature].astype(str))), 'csr', 'bool')
        base_predict_csr = sparse.hstack((base_predict_csr, cv.transform(predict_x[feature].astype(str))), 'csr',
                                         'bool')
    print('cv prepared !')

    sparse.save_npz( '../feature/base_train_csr.npz', base_train_csr)
    sparse.save_npz('../feature/base_predict_csr.npz', base_predict_csr)

# In[ ]:

#catvip=pd.read_hdf('../data/catvip.hdf', key='xunfei')['catvip'].values
allbase=sparse.vstack((base_train_csr,base_predict_csr),'csr')
#allbase=allbase[need,:][:,catvip]
#allbase=allbase.toarray()

# In[ ]:

all_df=sparse.hstack((allbase,all_df),'csr')

# In[ ]:

train_x =all_df[:-len(test)]
train_y = data[data.target != -1].target.values

# In[ ]:

test_x=all_df[-len(test):]

# In[ ]:

import gc
del allbase
gc.collect()
del all_df
gc.collect()
del data
gc.collect()

# In[ ]:

del all_data
gc.collect()

# In[ ]:

import datetime
import lightgbm as lgb
import gc
from scipy.sparse import csr_matrix
from sklearn.model_selection import KFold, StratifiedKFold
# sklearn风格lightgbm
import random
fea_rec=[]
fea_vip=[]
sub_preds = pd.DataFrame()

oof_predss = np.zeros(train_x.shape[0])
#argsDict={'bagging_fraction': 0.7441379310344828, 'bagging_freq': 2, 'feature_fraction': 0.8489655172413793, 'learning_rate': 0.07241379310344827, 'max_bin': 268, 'max_depth': -1, 'min_child_weight': 1.3684210526315788, 'min_data_in_bin': 10, 'min_split_gain': 0.21052631578947367, 'num_boost_round': 5000, 'num_leaves': 50, 'rands': 396, 'reg_alpha': 1.8421052631578947, 'reg_lambda': 7.894736842105263, 'scale_pos_weight': 1.0}

argsDict={'bagging_fraction': 0.98, 'bagging_freq': 8, 'feature_fraction': 0.9027586206896552, 'learning_rate': 0.052631578947368425, 'max_bin': 260, 'max_depth': -1, 'min_child_weight': 0.1724137931034483, 'min_data_in_bin': 7, 'min_split_gain': 0.1, 'num_boost_round': 5000, 'num_leaves': 56, 'rands': 152, 'reg_alpha': 6.842105263157895, 'reg_lambda': 13.974358974358974, 'scale_pos_weight': 1.0}

LGB=lgb.LGBMClassifier(
        num_leaves=argsDict["num_leaves"],
        max_depth=argsDict["max_depth"],
        learning_rate=argsDict["learning_rate"],
        n_estimators=argsDict['num_boost_round'],
         min_split_gain=argsDict["min_split_gain"],
        #min_child_samples=argsDict["min_data_in_leaf"],
        min_child_weight=argsDict["min_child_weight"],
        subsample=argsDict["bagging_fraction"],
        subsample_freq=argsDict["bagging_freq"],
        colsample_bytree=argsDict["feature_fraction"],
        reg_alpha=argsDict["reg_alpha"],
        reg_lambda=argsDict["reg_lambda"],
        scale_pos_weight=argsDict["scale_pos_weight"],

        is_training_metric= True,
        boosting_type='gbdt',
        metric='binary_logloss',
        n_jobs=9,
        #n_threads=10,
        seed=argsDict["rands"],
        drop_seed=argsDict["rands"],
        bagging_seed=argsDict["rands"],
        feature_fraction_seed=argsDict["rands"],
        random_state=argsDict["rands"],

        max_bin=argsDict["max_bin"],
        min_data_in_bin=argsDict["min_data_in_bin"],
    )
rands=random.randint(0,100)
skf = StratifiedKFold(n_splits=20, shuffle=True, random_state=argsDict["rands"])
tmprmselist=[]
treerec=[]
for ii,(train_index, test_index) in enumerate(skf.split(train_x, train_y)):

    x_train, x_val = train_x[train_index], train_x[test_index]
    y_train, y_val = train_y[train_index], train_y[test_index]
    print('fold:',ii)
    LGB.fit(x_train,y_train.reshape(-1,),eval_set=(x_val,y_val),eval_metric='logloss',early_stopping_rounds=100,verbose=100)
    tmplist=list(LGB.predict_proba(x_val)[:,1])
    oof_predss[test_index] =tmplist
        #print(LGB.evals_result_)
    tmprmse =np.mean(LGB.evals_result_['valid_0']['binary_logloss'])#np.sqrt(mean_squared_error(y_val, tmplist))
    print('best_itor:',LGB.best_iteration_,'    logloss:',tmprmse)
    treerec.append(LGB.best_iteration_)
    tmprmselist.append(tmprmse)

    sub_preds['estimators'+str(ii)]=(LGB.predict_proba(test_x,num_iteration=LGB.best_iteration_))[:,1]

    importance_dict={}
    for col,val in zip(range(train_x.shape[1]),LGB.feature_importances_):
        importance_dict[col]=val
    ser=pd.Series(importance_dict).sort_values(ascending=False)
    fea_vip+=list(ser[ser>3].index)
    del x_train
    gc.collect()
    del x_val
    gc.collect()

# In[ ]:

sub=test[['instance_id']]
sub['predicted_score']=(sub_preds.mean(axis=1)).values
org_test=pd.read_table('../data/round2_iflyad_test_feature.txt')
sub=org_test[['instance_id']].merge(sub,on='instance_id',how='left')
sub.to_csv('../subs/10151.csv',index=None)

# In[ ]:

oof_df=pd.DataFrame()
oof_df['instance_id']=train['instance_id'].values
oof_df['oof']=oof_predss.reshape(-1,)
oof_df.to_hdf('../oof/10151.csv',key='xunfei')


from serial import Serial
from config import (
    SINGLE_PRESS_DURATION,
    MIN,
    CENTER,
    MAX,
    Y,
    B,
    A,
    X,
    L,
    R,
    ZL,
    ZR,
    MINUS,
    PLUS,
    LCLICK,
    RCLICK,
    HOME,
    HAT_CENTER,
)
import time

def SplitHex(num):
    return [(num >> 8), (num & 0xFF)]

class Command:
    def __init__(
        self, lx=CENTER, ly=CENTER, rx=CENTER, ry=CENTER, hat=HAT_CENTER, button=0
    ):
        self.lx = lx
        self.ly = ly
        self.rx = rx
        self.ry = ry
        self.hat = hat
        self.button = button

    def __eq__(self, other):
        if isinstance(other, Command):
            return (
                self.lx == other.lx
                and self.ly == other.ly
                and self.rx == other.rx
                and self.ry == other.ry
                and self.hat == other.hat
                and self.button == other.button
            )
        return False

    def as_byte(self):
        button = SplitHex(self.button)
        return bytearray(
            [self.lx, self.ly, self.rx, self.ry, self.hat, button[0], button[1]]
        )

class Serial(Serial):
    def __init__(self, *args, **kwargs):
        self.p_command = None
        self.start_timer = 0
        self.stop_timer = 0
        self.csv_writer = None
        return super().__init__(*args, **kwargs)

    def attach_csv(self, csv_writer):
        self.csv_writer = csv_writer

    def my_write(self, command: Command):
        if self.p_command == None:
            self.p_command = command
            self.start_timer = time.time_ns()

        if self.p_command == command:
            self.counter = time.time()
            self.stop_timer = time.time_ns()
        else:
            self.csv_writer.writerow(
                [
                    self.p_command.lx,
                    self.p_command.ly,
                    self.p_command.rx,
                    self.p_command.ry,
                    self.p_command.hat,
                    self.p_command.button,
                    (self.stop_timer - self.start_timer),
                ]
            )
            self.start_timer = time.time_ns()
            self.stop_timer = self.start_timer
            self.p_command = command

        self.write(command.as_byte())


from rubicon.objc import objc_method

from toga.interface import DetailedList as DetailedListInterface

from .base import WidgetMixin
from ..libs import *
# from ..utils import process_callback

class TogaTableViewController(UITableViewController):
    @objc_method
    def numberOfSectionsInTableView_(self) -> int:
        return 1

    @objc_method
    def tableView_numberOfRowsInSection_(self, tableView, section: int) -> int:
        return len(self.data)

    @objc_method
    def tableView_cellForRowAtIndexPath_(self, tableView, indexPath):
        cell = tableView.dequeueReusableCellWithIdentifier_("row")
        if cell is None:
            cell = UITableViewCell.alloc().initWithStyle_reuseIdentifier_(UITableViewCellStyleDefault, "row")

        cell.textLabel.text = self.interface.data[indexPath.item]['description']
        return cell

    @objc_method
    def tableView_commitEditingStyle_forRowAtIndexPath_(self, tableView, editingStyle: int, indexPath):
        if editingStyle == UITableViewCellEditingStyleDelete:
            item = self.data[indexPath.row]

            if self.interface.on_delete:
                self.interface.on_delete(self.interface)

            del self.data[indexPath.row]

            paths = NSArray.alloc().initWithObjects_(indexPath, None)
            tableView.deleteRowsAtIndexPaths_withRowAnimation_(paths, UITableViewRowAnimationFade)

    @objc_method
    def refresh(self):
        if self.interface.on_refresh:
            self.interface.on_refresh(self.interface)
        self.refreshControl.endRefreshing()
        self.tableView.reloadData()

class DetailedList(DetailedListInterface, WidgetMixin):
    def __init__(self, id=None, data=None, on_delete=None, on_refresh=None, style=None):
        super().__init__(id=id, data=data, on_delete=on_delete, on_refresh=on_refresh, style=style)
        self._create()

    def create(self):
        self._controller = TogaTableViewController.alloc().init()
        self._controller.interface = self
        self._impl = self._controller.tableView

        self._controller.refreshControl = UIRefreshControl.alloc().init()
        self._controller.refreshControl.addTarget_action_forControlEvents_(
            self._controller,
            get_selector('refresh'),
            UIControlEventValueChanged
        )

        self._controller.data = self._config['data']

        # Add the layout constraints
        self._add_constraints()

    def _add(self, item):
        self.data.append(item)
        self._controller.tableView.reloadData()


import requests
import re
from requests.exceptions import RequestException
import json
from multiprocessing import Pool
import pymongo

client = pymongo.MongoClient('localhost')
db = client['maoyan']

def get_html(url):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 \
    (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        return response.text
    except RequestException:
        print('Failed to connect')

def parse_html(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?title="(.*?)".*?data-src="(.*?)".*?star">'
    +'(.*?)
.*?releasetime">(.*?)
.*?integer">(.*?).*?fraction">(.*?)
', re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            'index': item[0],
            'title': item[1],
            'image': item[2],
            'actor': item[3].strip()[3:],
            'time': item[4].strip()[5:],
            'score': item[5]+item[6]
        }

def save_to_file(content):
    with open('movie100.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + '\n')
        f.close()

def main(page):
    url = 'http://maoyan.com/board/4?offset=' + str(page)
    html = get_html(url)
    items = parse_html(html)
    n = 1
    for item in items:
        #save_to_file(item)
        if db["maoyan"].insert(item):
            print("第 %d 条保存到数据库成功" % n)
            n += 1

if __name__ == '__main__':
    #pool = Pool()
    #pool.map(main, [i*10 for i in range(10)])
    for i in range(10):
        main(i * 10)


X, Y, A, B = map(int, input().split())

ans = 0
for a in range(Y + 10):
    prd = X * pow(A, a)
    if prd >= Y:
        break
    b = max(0, (Y - prd - 1) // B)
    ans = max(ans, a + b)
print(ans)


from math import sqrt
t=int(input())
for i in range(t):
    a,b=input().split()
    x,y=int(a),int(b)

    def isPrime(n):
        flag=0
        for k in range(2,int(sqrt(n))+1):
            if n%k==0:
                flag=1
                break
        if flag==0:
            print(n)

    for j in range(x,y+1):
        if j == 1:
            pass
        elif j in [2,5]:
            print(j)
        elif j%10 in [1,3,7,9]:
            isPrime(j)
    print()


from time import time
from humanize import naturaldelta
from ensmallen_graph import EnsmallenGraph  # pylint: disable=no-name-in-module
import compress_json
import json
import numpy as np

start = time()
graph = EnsmallenGraph(
    edge_path="../graph/cooccurrence/edges.tsv",
    sources_column="subject",
    destinations_column="object",
    directed=False,
    validate_input_data=True
)
completed_graph = time() - start
start_walk = time()

walks = graph.walk(
    iterations=1,
    length=80,
    return_weight=1,
    explore_weight=1,
    change_node_type_weight=1,
    change_edge_type_weight=1
)
delta = time() - start
total_walk_time = time() - start_walk

mean_walks_length = np.mean([
    len(walk) for walk in walks
])

median_walks_length = np.median([
    len(walk) for walk in walks
])

degrees = [
    graph.degree(node)
    for node in range(graph.get_nodes_number())
]

response = {
    "directory": "cooccurrence",
    "total_required_time": delta,
    "building_graph_required_time": completed_graph,
    "random_walk_time": total_walk_time,
    "mean_walks_length": mean_walks_length,
    "median_walks_length": median_walks_length,
    "traps_rate": graph.traps_rate(),
    "mean_outbound_edges": np.mean(degrees),
    "median_outbound_edges": np.median(degrees),
    "nodes": graph.get_nodes_number(),
    "edges": graph.get_edges_number()
}

print(json.dumps(response, indent=4))

compress_json.dump(response, "time_required.json", json_kwargs={"indent": 4})


# 我采用requests库
import requests
import time

# 用来获取 时间戳
def gettime():
    return int(round(time.time() * 1000))

if __name__ == '__main__':
    # 用来自定义头部的
    headers = {}
    # 用来传递参数的
    keyvalue = {}
    # 目标网址(问号前面的东西)
    url = 'http://data.stats.gov.cn/easyquery.htm'

    # 头部的填充
    headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) ' \
                            'AppleWebKit/605.1.15 (KHTML, like Gecko) ' \
                            'Version/12.0 Safari/605.1.15'

    # 下面是参数的填充，参考图10
    keyvalue['m'] = 'QueryData'
    keyvalue['dbcode'] = 'hgnd'
    keyvalue['rowcode'] = 'zb'
    keyvalue['colcode'] = 'sj'
    keyvalue['wds'] = '[]'
    keyvalue['dfwds'] = '[{"wdcode":"zb","valuecode":"A0301"}]'
    keyvalue['k1'] = str(gettime())

    # 发出请求，使用get方法，这里使用我们自定义的头部和参数
    # r = requests.get(url, headers=headers, params=keyvalue)
    # 建立一个Session
    s = requests.session()
    # 在Session基础上进行一次请求
    r = s.get(url, params=keyvalue, headers=headers)
    # 打印返回过来的状态码
    print (r.status_code)
    # 修改dfwds字段内容
    keyvalue['dfwds'] = '[{"wdcode":"sj","valuecode":"2000"}]'
    # 再次进行请求
    r = s.get(url, params=keyvalue, headers=headers)
    # 此时我们就能获取到我们搜索到的数据了
    print (r.text)


import json
import traceback
import io
import pickle
import copy
from git import Repo
from . import app_logger
from web_app.auxiliary import exceptions

def load_model(model_spec):
    """Given a model and its version, load it"""

    #models_repo = r"/Users/davidlaredorazo/Documents/Projects/Rappi Challenge/models_and_data"
    models_repo = r"/models_and_data"

    model = None

    #Try to open repository
    try:
        repo = Repo(models_repo)
    except Exception as e:
        app_logger.error('Could not open repository')
        app_logger.error(traceback.format_exc())
        raise exceptions.FileError('Could not open repository')

    #Attempt to load models
    try:
        if not model_spec['model_tag']:
            raise exceptions.UnspecifiedModel('Model not specified')

        if model_spec['model_version']:
            commit = repo.commit(model_spec['model_version'])

            target_file = commit.tree / ('models/deploy/' + model_spec['model_tag'] + '.pkl')

            print(target_file)

            with io.BytesIO(target_file.data_stream.read()) as f:
                model = pickle.load(f)
        else:
            model = pickle.load(open(models_repo + '/models/deploy/' + model_spec['model_tag'] + '.pkl', 'rb'))
    except Exception as e:
        app_logger.error('Could not load model')
        app_logger.error(traceback.format_exc())
        raise exceptions.UnspecifiedModel('Could not load model')

    return model

def load_ml_models(app_root):
    """Load all the models specified in the models_list"""

    models_list = None
    models = {}
    model = None
    print(app_root)

    try:

        app_logger.error("Loading models from")
        app_logger.error(app_root / 'models_list.json')

        with open(app_root / 'models_list.json', 'r') as fp:
            models_list = json.load(fp)

        app_logger.error(models_list)

    except Exception as e:

        app_logger.error("Could not open models list file.")
        app_logger.error(traceback.format_exc())
        print("Could not open models list file. Check app_log file")

    for key in models_list:

        model_spec = models_list[key]
        model = load_model(model_spec)
        models[model_spec['model_tag'] + '/' +
               (model_spec['model_version'] if model_spec['model_version'] else 'latest')] = copy.deepcopy(model)

    return models


'''
@author:  etekinalp
@date:    Aug 26, 2014
@mail:    e.tekinalp@icloud.com
@brief:   This module setups a plugin
'''

from maya import cmds, mel
from goe_plugins import plugin_master
reload(plugin_master)

class Plug(plugin_master.PluginSetup):
    """
    Subclass PluginSetup and setup the plugin test environment.
    """

    def __init__(self, plugin, name):
        """
        @param plugin(string): Plugin name without .so or .py suffix
        @param name(string): Name of plugin call
        """

        super(Plug, self).__init__(plugin, 'so', True, True)

        # args
        self.plugin = plugin
        self.name = name

        # methods
        self._setup_plugin()
    # END __init__()

    def _setup_plugin(self):
        cmds.createNode(self.name)
    # END _setup_plugin()
# END Plug()

Plug("curveramp", "curveRamp")


bdata = bytes(range(0, 256))
print(len(bdata))

fout = open('bfile', 'wb')
print(fout.write(bdata))
fout.close()

fout = open('bfile2', 'wb')
size = len(bdata)
offset = 0
chunk = 100
while True:
    if offset > size:
        break
    fout.write(bdata[offset:offset+chunk])
    offset += chunk

fout.close()


from django.test import TestCase,Client
from .models import Airport
from .models import Flight

# Create your tests here.

class ModelsTestCase(TestCase):
    def setUp(self):

        a1 =  Airport.objects.create(code="AAA",city="City A")
        a2 = Airport.objects.create(code= "BBB", city = "City B")

        Flight.objects.create(origin=a1,destination=a2,duration=100)
        Flight.objects.create(origin=a1,destination=a1,duration=200)
        Flight.objects.create(origin=a1,destination=a2,duration=-100)

    def test_departure_count(self):
        a = Airport.objects.get(code ="AAA")
        self.assertEqual(a.departures.count(),3)

    def test_arrival_count(self):
        a = Airport.objects.get(code ="AAA")
        self.assertEqual(a.arrivals.count(),1)

    def test_valid_flight(self):
        a1 = Airport.objects.get(code ="AAA")
        a2 = Airport.objects.get(code ="BBB")
        f = Flight.objects.get(origin=a1,destination= a2,duration=100)
        self.assertTrue(f.is_valid_flight())

    def test_invalid_flight_destination(self):
        a1 = Airport.objects.get(code ="AAA")
        f = Flight.objects.get(origin=a1,destination= a1)
        self.assertFalse(f.is_valid_flight())

    def test_invalid_flight_duration(self):
        a1 = Airport.objects.get(code ="AAA")
        a2 = Airport.objects.get(code ="BBB")
        f = Flight.objects.get(origin=a1,destination= a2,duration=-100)
        self.assertFalse(f.is_valid_flight())

    def test_index(self):
        c = Client()
        response = c.get("/")
        self.assertEqual(response.status_code,200)
        self.assertEqual(response.context["flights"].count(),3)

    def test_valid_flight_page(self):
        a1 = Airport.objects.get(code = "AAA")
        f =  Flight.objects.get(origin = a1, destination = a1)
        c = Client()
        response = c.get(f"/{f.id}")
        self.assertEqual(response.status_code,200)

   #def test_invalid_flight(self):
    #    max_id = Flight.objects.all().aggregate(Max("id")["id_max"])
     #   c = Client()
      #  response = c.get(f"/{max_id +1}")
       # self.assertEqual(response.status_code,404)