diff --git a/requirements.txt b/requirements.txt index 1ec63236d555..b38be17ab9e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,11 +5,11 @@ # To install dev requirements: inv requirements --dev # To install release requirements: inv requirements --release -future==0.18.2 +future==0.18.3 invoke==0.15.0 -Werkzeug==1.0.0 -Flask==1.0 -Mako==1.0.7 +Werkzeug==2.2.3 +Flask==2.2.5 +Mako==1.2.2 Markdown==3.3.7 WTForms==1.0.4 # Fork of celery 4.1.1 with https://github.com/celery/celery/pull/4278 backported, @@ -19,7 +19,7 @@ WTForms==1.0.4 git+https://github.com/cos-forks/celery@v4.1.1+cos0 kombu==4.2.0 itsdangerous==1.1.0 -lxml==4.6.5 +lxml==4.9.1 mailchimp3==3.0.18 nameparser==0.5.3 bcrypt==3.1.4 @@ -41,29 +41,29 @@ tqdm==4.28.1 # Python markdown extensions for comment emails git+https://github.com/Johnetordoff/mdx_del_ins.git@django-3 -certifi==2020.12.5 +certifi==2023.7.22 sendgrid==1.5.13 requests>=2.21.0 -urllib3==1.26.5 +urllib3==1.26.17 oauthlib==2.0.6 requests-oauthlib==0.8.0 raven==6.4.0 django-redis==5.2.0 # API requirements -Django==3.2.17 +Django==3.2.20 djangorestframework==3.13.1 django-cors-headers==3.10.1 djangorestframework-bulk==0.2.1 hashids==1.2.0 -pyjwt==1.5.3 +pyjwt==2.4.0 django-celery-beat==1.1.1 # BSD 3 Clause -django-celery-results==1.0.1 +django-celery-results==2.4.0 pyjwe==1.0.0 # Required by pyjwe and ndg-httpsclient # Building wheel for cryptography >= 3.4.0 requires a Rust version incompatible with Docker base image. -cryptography==3.3.2 +cryptography==41.0.4 jsonschema==3.1.1 django-guardian==2.4.0 diff --git a/requirements/release.txt b/requirements/release.txt index 1d0cdddebba4..9cd1a1f8ebbe 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -5,4 +5,4 @@ # newrelic APM agent newrelic==7.16.0.178 # uwsgi -uwsgi==2.0.16 +uwsgi==2.0.22 diff --git a/scripts/EGAP/EGAP_tests.py b/scripts/EGAP/EGAP_tests.py deleted file mode 100644 index baf9ef351ce2..000000000000 --- a/scripts/EGAP/EGAP_tests.py +++ /dev/null @@ -1,187 +0,0 @@ -import unittest -from jsonschema.exceptions import ValidationError -from create_EGAP_json import (schema_to_spreadsheet_mapping, - make_project_dict, - make_registration_dict, - other_mapping, -) - -HEADER_ROW = ['POST DATE', - 'ID', - 'STATUS', - 'TITLE', - 'B2 AUTHORS', - 'EMAIL', - 'B3 ACKNOWLEDGEMENTS', - 'B4 FACULTY MEMBER?', - 'B5 PROSPECTIVE OR RETROSPECTIVE?', - 'B6 EXPERIMENTAL STUDY?', - 'B7 DATE OF START OF STUDY', - 'B8 GATE DATE', - 'B8 FORMERLY GATED UNTIL', - 'B9 PRESENTED AT EGAP MEETING?', - 'B10 PRE-ANALYSIS PLAN WITH REGISTRATION?', - 'C1 BACKGROUND', - 'C2 HYPOTHESES', - 'C3 TESTING PLAN', - 'C4 COUNTRY', - 'C5 SAMPLE SIZE', - 'C6 POWER ANALYSIS?', - 'C7 IRB APPROVAL?', - 'C8 IRB NUMBER', - 'C9 DATE OF IRB APPROVAL', - 'C10 INTERVENTION IMPLEMENTER', - 'C11 REMUNERATION?', - 'C12 PUBLICATION AGREEMENT?', - 'C13 JEL CODES', - 'METHODOLOGY', - 'POLICY'] - -TEST_ROW_WITH_OTHER = ['03/05/2017 - 17:00', - '20170305AA', - 'Status is not saved, so this field doesnt matter', - 'The members of Nsync', - 'Justin Timberlake | Joey Fatone | Lance Bass', - 'doesnt@matter.com', - 'We acknolowledge Chris Kirkpatrick', - 'Justin Timberlake is a faculty Member', - 'This is my other response for prospective', - 'Yes', - '05/01/2017', - '05/01/2020', - '', - 'No', - 'No', - 'Test background', - 'test hypothesis', - 'This is my testing plan', - 'Switzerland', - '3242', - 'This is a power analysis other response', - 'This is an other irb response', - '343434', - '03/06/2017', - 'This is an other intervention response', - 'This is an other renumeration response', - 'This is an other publication agreement response', - 'Jel Code', - 'Survey Methodology', - 'Gender'] - -# Testing row with missing required fields. i.e. Hypothesis, Background, testing plan. -TEST_ROW_WITH_MISSING = ['03/05/2017 - 17:00', - '20170305AA', - 'Status is not saved, so this field doesnt matter', - 'The members of Nsync', - 'Justin Timberlake | Joey Fatone | Lance Bass', - 'doesnt@matter.com', - 'We acknolowledge Chris Kirkpatrick', - 'Justin Timberlake is a faculty Member', - 'This is my other response for prospective', - 'Yes', - '05/01/2017', - '05/01/2020', - '', - 'No', - 'No', - '', - '', - '', - 'Switzerland', - '3242', - 'This is a power analysis other response', - 'This is an other irb response', - '343434', - '03/06/2017', - 'This is an other intervention response', - 'This is an other renumeration response', - 'This is an other publication agreement response', - 'Jel Code', - 'Survey Methodology', - 'Gender'] - -TEST_ROW_WITH_OTHER_AUTHORS = [ - {'name': 'Justin Timberlake', 'email': 'jt@gmail.com'}, - {'name': 'Joey Fatone'}, - {'name': 'Lance Bass', 'email': 'lBass@gmail.com'}] - -TEST_ROW = ['05/05/2018 - 17:00', - '20180505AA', - 'Status is not saved, so this field doesnt matter', - 'The members of Backstreet boys', - 'Nick Carter | Brian Littrell, Ph.D. | AJ McLean | U.S. Agency Bureau, Department of Agency affairs (DOAA)', - 'doesnt@matter.com', - 'We acknolowledge Chris Kirkpatrick', - 'Yes', - 'Registration prior to any research activities', - 'Yes', - '05/01/2017', - '05/01/2020', - '', - 'No', - 'No', - 'Test background', - 'test hypothesis', - 'This is my testing plan', - 'Switzerland', - '3242', - 'Yes', - 'Yes', - '343434', - '03/06/2017', - 'Researchers', - 'Yes', - 'Yes', - 'Jel Code', - 'Survey Methodology', - 'Gender'] - -TEST_ROW_AUTHORS = [ - {'name': 'Nick Carter', 'email': 'nickc@gmail.com'}, - {'name': 'Brian Littrell, Ph.D.'}, - {'name': 'AJ McLean', 'email': 'AJML@gmail.com'}, - {'name': 'U.S. Agency Bureau, Department of Agency affairs (DOAA)', 'email': 'DOAA@UAB.gov'}] - -class TestProjectDict(unittest.TestCase): - - def test_row_with_other(self): - project_dict = make_project_dict(TEST_ROW_WITH_OTHER, TEST_ROW_WITH_OTHER_AUTHORS, HEADER_ROW) - self.assertEqual(project_dict['title'], TEST_ROW_WITH_OTHER[3]) - self.assertEqual(project_dict['contributors'], TEST_ROW_WITH_OTHER_AUTHORS) - self.assertEqual(project_dict['post-date'], TEST_ROW_WITH_OTHER[0]) - self.assertEqual(project_dict['id'], TEST_ROW_WITH_OTHER[1]) - - def test_row(self): - project_dict = make_project_dict(TEST_ROW, TEST_ROW_AUTHORS, HEADER_ROW) - self.assertEqual(project_dict['title'], TEST_ROW[3]) - self.assertEqual(project_dict['contributors'], TEST_ROW_AUTHORS) - self.assertEqual(project_dict['post-date'], TEST_ROW[0]) - self.assertEqual(project_dict['id'], TEST_ROW[1]) - -class TestRegistrationDict(unittest.TestCase): - - def run_registration_test(self, row, header_row): - project_dict = make_registration_dict(row, header_row, row[1]) - for question_dict in schema_to_spreadsheet_mapping: - question_key = question_dict.keys()[0] - spreadsheet_column = question_dict[question_key] - column_index = header_row.index(spreadsheet_column) - if type(project_dict[question_key]['value']) == list: - field_val = project_dict[question_key]['value'][0] - else: - field_val = project_dict[question_key]['value'] - if row[column_index] != field_val and question_key in other_mapping: - self.assertEqual(project_dict[question_key]['value'], 'Other (describe in text box below)') - field_val = project_dict[other_mapping[question_key]]['value'] - self.assertEqual(row[column_index], field_val) - else: - self.assertEqual(row[column_index], field_val) - - def test_row_with_other(self): - self.run_registration_test(TEST_ROW_WITH_OTHER, HEADER_ROW) - - def test_row(self): - self.run_registration_test(TEST_ROW, HEADER_ROW) - - def test_row_with_errors(self): - self.assertRaises(Exception, make_registration_dict, TEST_ROW_WITH_MISSING, HEADER_ROW, TEST_ROW_WITH_MISSING[1]) diff --git a/scripts/EGAP/__init__.py b/scripts/EGAP/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scripts/EGAP/create_EGAP_json.py b/scripts/EGAP/create_EGAP_json.py deleted file mode 100644 index 37c19e2fcd16..000000000000 --- a/scripts/EGAP/create_EGAP_json.py +++ /dev/null @@ -1,565 +0,0 @@ -import logging -import csv -import datetime -import json -import os -import shutil -import jsonschema -import argparse - -from jsonschema.exceptions import ValidationError - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - -parser = argparse.ArgumentParser() -parser.add_argument('-a', '--authorsource', help='Specify the source file for the author csv file') -parser.add_argument('-r', '--registrysource', help='Specify the source file for the registrty csv file') -parser.add_argument('-t', '--target', help='Specify the target directory of the registry directories') -parser.add_argument('-d', '--dry', action='store_true', help='Dry run: Have the script delete the target directory after completion') - -schema_to_spreadsheet_mapping = [ - {'q1': 'TITLE'}, - {'q3': 'ID'}, - {'q4': 'POST DATE'}, - {'q5': 'B3 ACKNOWLEDGEMENTS'}, - {'q6': 'B4 FACULTY MEMBER?'}, - {'q8': 'B5 PROSPECTIVE OR RETROSPECTIVE?'}, - {'q10': 'B6 EXPERIMENTAL STUDY?'}, - {'q11': 'B7 DATE OF START OF STUDY'}, - {'q12': 'B8 GATE DATE'}, - {'q13': 'B9 PRESENTED AT EGAP MEETING?'}, - {'q14': 'B10 PRE-ANALYSIS PLAN WITH REGISTRATION?'}, - {'q15': 'C1 BACKGROUND'}, - {'q16': 'C2 HYPOTHESES'}, - {'q17': 'C3 TESTING PLAN'}, - {'q18': 'C4 COUNTRY'}, - {'q19': 'C5 SAMPLE SIZE'}, - {'q20': 'C6 POWER ANALYSIS?'}, - {'q22': 'C7 IRB APPROVAL?'}, - {'q24': 'C8 IRB NUMBER'}, - {'q25': 'C9 DATE OF IRB APPROVAL'}, - {'q26': 'C10 INTERVENTION IMPLEMENTER'}, - {'q28': 'C11 REMUNERATION?'}, - {'q30': 'C12 PUBLICATION AGREEMENT?'}, - {'q32': 'C13 JEL CODES'}, - {'q33': 'METHODOLOGY'}, - {'q34': 'POLICY'}, -] - -# Any multiple choice questions where "Other" is a possible response, have subsequent "Other" -# question to log that response. If multiple choice question value is invalid, -# attempt to log the value in the corresponding "Other" question response. -other_mapping = { - 'q6': 'q7', - 'q8': 'q9', - 'q20': 'q21', - 'q22': 'q23', - 'q26': 'q27', - 'q28': 'q29', - 'q30': 'q31' -} - - -here = os.path.split(os.path.abspath(__file__))[0] - - -def from_json(fname): - with open(os.path.join(here, fname)) as f: - return json.load(f) - - -def ensure_schema_structure(schema): - schema['pages'] = schema.get('pages', []) - schema['title'] = schema['name'] - schema['version'] = schema.get('version', 1) - return schema - - -def create_file_tree_and_json(author_source, registry_source, target): - # Things this function needs to do: - # For each row in the registry function, create a directory. - # Create two JSON files, one project json with ID, Title, Postdate, and authors listed - # with emails. And another with all the key value pairs for the registry meta. - top_dir = target - logger.info('Creating EGAP directory at {}'.format(top_dir)) - os.mkdir(top_dir) - author_list = create_author_dict(author_source) - with open(registry_source, 'rt', encoding='utf-8-sig') as csv_registry_file: - csv_reader = csv.reader(csv_registry_file, delimiter=',') - header_row = next(csv_reader) - normalized_header_row = [col_header.strip() for col_header in header_row] - logger.info('Debug data') - logger.info('Header row: {}'.format(header_row)) - logger.info('Normalized header row: {}'.format(normalized_header_row)) - - id_index = normalized_header_row.index('ID') - for line in csv_reader: - row = [cell for cell in line] - project_id = row[id_index] - root_directory = os.path.join(top_dir, project_id) - os.mkdir(root_directory) - data_directory = os.path.join(root_directory, 'data') - os.mkdir(data_directory) - os.mkdir(os.path.join(data_directory, 'nonanonymous')) - project_dict = make_project_dict(row, author_list, normalized_header_row) - make_json_file(root_directory, project_dict, 'project') - try: - registration_dict = make_registration_dict(row, normalized_header_row, project_id) - except Exception: - logger.warning('Error creating directory for {}'.format(project_id)) - shutil.rmtree(root_directory) - continue - make_json_file(root_directory, registration_dict, 'registration') - logger.info('Successfully created directory for {}'.format(project_id)) - - - -def create_author_dict(source): - # Reads in author CSV and returns a list of dicts with names and emails of EGAP Authors - authors = [] - with open(source, 'rt', encoding='utf-8-sig') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - header_row = next(csv_reader) - normalized_header_row = [col_header.strip() for col_header in header_row] - logger.info('Debug data') - logger.info('Header row: {}'.format(header_row)) - logger.info('Normalized header row: {}'.format(normalized_header_row)) - name_index = normalized_header_row.index('Name') - email_index = normalized_header_row.index('Email') - for line in csv_reader: - row = [cell for cell in line] - logger.info('Adding user: ' + row[name_index]) - if row[email_index] != '': - author_dict = {'name': row[name_index].strip(), 'email': row[email_index]} - else: - author_dict = {'name': row[name_index].strip()} - authors.append(author_dict) - return authors - - -def make_project_dict(row, author_list, normalized_header_row): - project = {} - title_index = normalized_header_row.index('TITLE') - id_index = normalized_header_row.index('ID') - postdate_index = normalized_header_row.index('POST DATE') - contributors_index = normalized_header_row.index('B2 AUTHORS') - project['id'] = row[id_index] - project['title'] = row[title_index] - project['post-date'] = row[postdate_index] - - authors = row[contributors_index] - - authors = authors.split('|') - project['contributors'] = [] - author_name_list = [author['name'] for author in author_list] - for author in authors: - author = author.strip() - if author: - if author not in author_name_list: - logger.warning('Author {} not in Author spreadsheet for project {}.'.format(author,row[id_index])) - project['contributors'].append({'name': author}) - else: - author_list_index = author_name_list.index(author) - project['contributors'].append(author_list[author_list_index]) - return project - - -def make_registration_dict(row, normalized_header_row, project_id): - registration = {} - - for question in schema_to_spreadsheet_mapping: - qid = list(question.keys())[0] - column_name = list(question.values())[0] - value = build_question_response(normalized_header_row, row, qid, column_name) - if value['value'] == '': - continue - validated_qid, other_response = validate_response(qid, value) - - if other_response == 'q26': - if other_response: - responses = [] - if value['value'][0].startswith('Researchers,'): - responses.append('Researchers') - responses.append('Third party (describe in text box below)') - registration[other_response] = build_nested_response(responses) - value['value'] = value['value'][0] - elif other_response: - registration[other_response] = build_nested_response('Other (describe in text box below)') - registration[validated_qid] = value - # q35 and q36 are required questions at the end of the schema, certification and - # confirmation questions. Just marking as agree - - registration['q35'] = build_nested_response('Agree') - registration['q36'] = build_nested_response('Agree') - validate_all_responses(registration, project_id) - return registration - - -def make_json_file(filepath, data, json_type): - if json_type == 'project': - filepath = filepath + '/project.json' - if json_type == 'registration': - filepath = filepath + '/registration-schema.json' - with open(filepath, 'w') as outfile: - json.dump(data, outfile) - - -def build_question_response(header_row, row, question_key, column_title): - """Format the question's response to go in the registration_metadata - :param header_row: Header row in spreadsheet - :param row: Row in spreadsheet - :param question_key: string, Official question key as part of schema - :param column_title: string, Corresponding question_key column title in EGAP spreadsheet - """ - index = header_row.index(column_title) - value = clean_value(row[index]) - # Spreadsheet has these as comma-separated values, but looking for array - if question_key == 'q26': - value = [value] - if question_key in ['q33', 'q34']: - value = value.split(', ') - return build_nested_response(value) - - -def clean_value(value): - """Clean spreadsheet values of issues that will affect validation """ - if value == 'n/a': - return 'N/A' - elif value == 'Design was registered before field was added': - return '' - return value - -def build_nested_response(value): - return { - 'comments': [], - 'extra': [], - 'value': value - } - - -def base_metaschema(metaschema): - json_schema = { - 'type': 'object', - 'description': metaschema['description'], - 'title': metaschema['title'], - 'additionalProperties': False, - 'properties': { - } - } - return json_schema - - -def get_required(question): - """ - Returns True if metaschema question is required. - """ - required = question.get('required', False) - if not required: - properties = question.get('properties', False) - if properties and isinstance(properties, list): - for item, property in enumerate(properties): - if isinstance(property, dict) and property.get('required', False): - required = True - break - return required - - -COMMENTS_SCHEMA = { - 'type': 'array', - 'items': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'seenBy': { - 'type': 'array', - }, - 'canDelete': {'type': 'boolean'}, - 'created': {'type': 'string'}, - 'lastModified': {'type': 'string'}, - 'author': {'type': 'string'}, - 'value': {'type': 'string'}, - 'isOwner': {'type': 'boolean'}, - 'getAuthor': {'type': 'string'}, - 'user': { - 'type': 'object', - 'additionalProperties': True, - 'properties': { - 'fullname': {'type': 'string'}, - 'id': {'type': 'integer'} - } - }, - 'saved': {'type': 'boolean'}, - 'canEdit': {'type': 'boolean'}, - 'isDeleted': {'type': 'boolean'} - } - } -} - - -def get_options_jsonschema(options, required): - """ - Returns multiple choice options for schema questions - """ - for item, option in enumerate(options): - if isinstance(option, dict) and option.get('text'): - options[item] = option.get('text') - value = {'enum': options} - - if not required and '' not in value['enum']: # Non-required fields need to accept empty strings as a value. - value['enum'].append('') - - return value - - -def get_object_jsonschema(question, required_fields, is_reviewer, is_required): - """ - Returns jsonschema for nested objects within schema - """ - object_jsonschema = { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - - } - } - required = [] - properties = question.get('properties') - if properties: - for property in properties: - if property.get('required', False) and required_fields: - required.append(property['id']) - values = extract_question_values(property, required_fields, is_reviewer, is_required) - object_jsonschema['properties'][property['id']] = { - 'type': 'object', - 'additionalProperties': False, - 'properties': values - } - if required_fields: - object_jsonschema['properties'][property['id']]['required'] = ['value'] - if required_fields and is_required: - object_jsonschema['required'] = required - - return object_jsonschema - - -OSF_UPLOAD_EXTRA_SCHEMA = { - 'type': 'array', - 'items': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'data': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'kind': {'type': 'string'}, - 'contentType': {'type': 'string'}, - 'name': {'type': 'string'}, - 'extra': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'downloads': {'type': 'integer'}, - 'version': {'type': 'integer'}, - 'latestVersionSeen': {'type': 'string'}, - 'guid': {'type': 'string'}, - 'checkout': {'type': 'string'}, - 'hashes': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'sha256': {'type': 'string'}, - 'md5': {'type': 'string'} - } - } - } - }, - 'materialized': {'type': 'string'}, - 'modified': {'type': 'string'}, - 'nodeId': {'type': 'string'}, - 'etag': {'type': 'string'}, - 'provider': {'type': 'string'}, - 'path': {'type': 'string'}, - 'nodeUrl': {'type': 'string'}, - 'waterbutlerURL': {'type': 'string'}, - 'resource': {'type': 'string'}, - 'nodeApiUrl': {'type': 'string'}, - 'type': {'type': 'string'}, - 'accept': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'acceptedFiles': {'type': 'boolean'}, - 'maxSize': {'type': 'integer'}, - } - }, - 'links': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'download': {'type': 'string'}, - 'move': {'type': 'string'}, - 'upload': {'type': 'string'}, - 'delete': {'type': 'string'} - } - }, - 'permissions': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'edit': {'type': 'boolean'}, - 'view': {'type': 'boolean'} - } - }, - 'created_utc': {'type': 'string'}, - 'id': {'type': 'string'}, - 'modified_utc': {'type': 'string'}, - 'size': {'type': 'integer'}, - 'sizeInt': {'type': 'integer'}, - } - }, - 'fileId': {'type': ['string', 'object']}, - 'descriptionValue': {'type': 'string'}, - 'sha256': {'type': 'string'}, - 'selectedFileName': {'type': 'string'}, - 'nodeId': {'type': 'string'}, - 'viewUrl': {'type': 'string'} - } - } -} - - -def extract_question_values(question, required_fields, is_reviewer, is_required): - """ - Pulls structure for 'value', 'comments', and 'extra' items - """ - response = { - 'value': {'type': 'string'}, - 'comments': COMMENTS_SCHEMA, - 'extra': {'type': 'array'} - } - if question.get('type') == 'object': - response['value'] = get_object_jsonschema(question, required_fields, is_reviewer, is_required) - elif question.get('type') == 'choose': - options = question.get('options') - if options: - enum_options = get_options_jsonschema(options, is_required) - if question.get('format') == 'singleselect': - response['value'] = enum_options - elif question.get('format') == 'multiselect': - response['value'] = {'type': 'array', 'items': enum_options} - elif question.get('type') == 'osf-upload': - response['extra'] = OSF_UPLOAD_EXTRA_SCHEMA - - if is_reviewer: - del response['extra'] - if not question.get('type') == 'object': - del response['value'] - - return response - - -def create_jsonschema_from_metaschema(metaschema, required_fields=False, is_reviewer=False): - """ - Creates jsonschema from registration metaschema for validation. - - Reviewer schemas only allow comment fields. - """ - json_schema = base_metaschema(metaschema) - required = [] - - for page in metaschema['pages']: - for question in page['questions']: - is_required = get_required(question) - if is_required and required_fields: - required.append(question['qid']) - json_schema['properties'][question['qid']] = { - 'type': 'object', - 'additionalProperties': False, - 'properties': extract_question_values(question, required_fields, is_reviewer, is_required) - } - if required_fields: - json_schema['properties'][question['qid']]['required'] = ['value'] - - if required and required_fields: - json_schema['required'] = required - - return json_schema - - -def validate_response(qid, value): - """Validate question response - - Validating each question response individually. If there is an error, we will - attempt to add the value to the corresponding "Other" block. Return that question id instead. - - For example, q6 is a multiple choice question, with "Other" as a choice. If text is entered - for q6 that does not match one of the multiple choice answers, assuming that this is "other" - text, and this response should go to the corresponding q7 question. q6 will be marked - as "Other" - - :param qid: string, question id from schema - :param value: question response - :param draft: DraftRegistration - :return qid: tuple, (qid corresponding to value, optional "Other" qid) - """ - temporary_check = {} - temporary_check[qid] = value - egap_schema = ensure_schema_structure(from_json('egap-registration-3.json')) - schema = create_jsonschema_from_metaschema(egap_schema, - required_fields=False, - is_reviewer=False) - - try: - json_schema = jsonschema.validate(temporary_check, schema) - except ValidationError as exc: - if qid in other_mapping: - return other_mapping[qid], qid - else: - raise Exception(exc) - return qid, None - -def validate_all_responses(value, project_id): - egap_schema = ensure_schema_structure(from_json('egap-registration-3.json')) - schema = create_jsonschema_from_metaschema(egap_schema, - required_fields=True, - is_reviewer=False) - - try: - json_schema = jsonschema.validate(value, schema) - except ValidationError as exc: - with open('errors.txt', 'a+') as error_file: - error_file.write(', '+project_id) - raise Exception(exc) - - -def main(default_args=False): - if default_args: - args = parser.parse_args(['--source', 'default', '--target', 'default']) - else: - args = parser.parse_args() - - author_source = args.authorsource - registry_source = args.registrysource - target_directory = args.target - dry_run = args.dry - - if not author_source: - author_source = 'EGAP_author_emails.csv' - - if not registry_source: - registry_source = 'EGAP_registry_for_OSF.csv' - - if not target_directory: - target_directory = 'EGAP_data_{}'.format(datetime.datetime.now().strftime('%m-%d-%Y')) - - create_file_tree_and_json(author_source, registry_source, target_directory) - - if dry_run: - shutil.rmtree(target_directory) - raise RuntimeError('Dry run, file tree being deleted.') - - -if __name__ == '__main__': - - main(default_args=False) diff --git a/scripts/EGAP/egap-registration-3.json b/scripts/EGAP/egap-registration-3.json deleted file mode 100644 index 9807e2d8231d..000000000000 --- a/scripts/EGAP/egap-registration-3.json +++ /dev/null @@ -1,385 +0,0 @@ -{ - "name": "EGAP Registration", - "version": 3, - "config": { - "hasFiles": true - }, - "description": "The EGAP registry focuses on designs for experiments and observational studies in governance and politics.", - "pages": [{ - "id": "page1", - "title": "General Information About the Study", - "questions": [{ - "qid": "q1", - "nav": "Title", - "type": "string", - "format": "text", - "title": "Title of Study", - "description": "Provide the working title of your study.", - "required": true - }, - { - "qid": "q3", - "nav": "EGAP Registration ID", - "title": "EGAP Registration ID", - "format": "textarea", - "required": true - }, - { - "qid": "q4", - "nav": "Timestamp", - "title": "Timestamp of original registration", - "format": "textarea", - "required": true - }, - { - "qid": "q5", - "nav": "Acknowledgements", - "title": "Acknowledgements", - "type": "string", - "format": "textarea", - "required": false - }, - { - "qid": "q6", - "title": "Is one of the study authors a university faculty member?", - "nav": "University Faculty Member?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A", - "Other (describe in text box below)" - ], - "description": "Please choose one" - }, - { - "qid": "q7", - "title": "Other author affiliation", - "format": "textarea", - "required": false, - "description": "Other author affiliation (additional information if 'other' was selected above)" - - }, - { - "qid": "q8", - "title": "Is this Registration Prospective or Retrospective?", - "nav": "Prospective or Retrospective?", - "type": "choose", - "format": "singleselect", - "options": [ - "Registration prior to any research activities", - "Registration prior to assignment of treatment", - "Registration prior to realization of outcomes", - "Registration prior to researcher access to outcome data", - "Registration prior to researcher analysis of outcome data", - "Registration after researcher analysis of outcome data", - "N/A", - "Other (describe in text box below)" - ], - "description": "Please choose one" - }, - { - "qid": "q9", - "title": "Other description of registration timing", - "format": "textarea", - "required": false, - "description": "Other (additional information if 'other' was selected above)" - }, - { - "qid": "q10", - "title": "Is this an experimental study?", - "description": "With “experimental” defined as random assignment of units to treatment and control conditions.", - "nav": "Experimental study?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A" - ] - }, - { - "qid": "q11", - "title": "Date of start of study", - "nav": "Date of start of study", - "type": "string", - "format": "text", - "description": "Understood as first date of treatment assignment or equivalent for observational study", - "help": "E.g., 06/02/2018" - }, - { - "qid": "q12", - "title": "B8 Gate Date", - "nav": "Gate Date?", - "type": "string", - "format": "text", - "description": "Gating is discouraged, but if necessary, EGAP policy limits the gate range to 18 months maximum.", - "help": "E.g., 06/02/2018" - }, - { - "qid": "q13", - "title": "Was this design presented at an EGAP meeting?", - "nav": "Presented at an EGAP meeting?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A" - - ] - }, - { - "qid": "q14", - "title": "Is there a pre-analysis plan associated with this registration?", - "nav": "Pre-analysis plan associated with this registration?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A" - ], - "description": "If so, please attach it in the Additional Documentation section on the final screen." - } - ] - }, - { - "id": "page2", - "title": "Registration Data", - "questions": [{ - "qid": "q15", - "nav": "Background and explanation of rationale.", - "title": "Background and explanation of rationale.", - "format": "textarea", - "required": true, - "description": "Brief description of goals of project. If you are also attaching a pre-analysis plan, please refrain from simply copying and pasting a section from your plan here. If possible, please also avoid saying \"see attached pre-analysis plan,\" as it renders the search functionality less useful. Rather, please provide a short (1-2 paragraph) summary of the project background." - }, - { - "qid": "q16", - "nav": "Background and explanation of rationale.", - "title": "What are the hypotheses to be tested/quantities of interest to be estimated?", - "format": "textarea", - "required": true, - "description": "Please list the hypotheses including hypotheses on heterogeneous effects. If you are also attaching a pre-analysis plan, please refrain from simply copying and pasting a section from your plan here. If possible, please also avoid saying \"see attached pre-analysis plan,\" as it renders the search functionality less useful. Rather, please provide a short (1-2 paragraph) summary of project hypotheses." - }, - { - "qid": "q17", - "nav": "How will these hypotheses be tested?", - "title": "How will these hypotheses be tested?", - "format": "textarea", - "required": true, - "description": "Brief description of your methodology. If you are also attaching a pre-analysis plan, please refrain from simply copying and pasting a section from your plan here. If possible, please also avoid saying \"see attached pre-analysis plan,\" as it renders the search functionality less useful. Rather, please provide a short (1-2 paragraph) summary of project methodology." - }, - { - "qid": "q18", - "title": "Country", - "nav": "Country", - "type": "string", - "format": "text", - "help": "Comma separated names of countries (e.g. Canada, United States, Mexico)" - }, - { - "qid": "q19", - "title": "Sample Size (# of Units)", - "nav": "Sample Size", - "type": "string", - "format": "text" - }, - { - "qid": "q20", - "title": "Was a power analysis conducted prior to data collection?", - "nav": "Power analysis conducted prior to data collection?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A", - "Other (describe in text box below)" - ] - }, - { - "qid": "q21", - "title": "Other power analysis information", - "format": "textarea", - "description": "Other (additional information if 'other' was selected above)", - "required": false - }, - { - "qid": "q22", - "title": "Has this research received Institutional Review Board (IRB) or ethics committee approval?", - "nav": "Review Board (IRB) or ethics committee approval?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A", - "Other (describe in text box below)" - ] - }, - { - "qid": "q23", - "title": "Other IRB information", - "description": "Other (additional information if 'other' was selected above)", - "format": "textarea", - "required": false - }, - { - "qid": "q24", - "title": "IRB Number", - "nav": "IRB Number", - "type": "string", - "format": "text" - }, - { - "qid": "q25", - "title": "Date of IRB Approval", - "nav": "IRB Number", - "type": "string", - "format": "text" - }, - { - "qid": "q26", - "title": "Will the intervention be implemented by the researcher or a third party? If a third party, please provide the name.", - "nav": "Review Board (IRB) or ethics committee approval?", - "type": "choose", - "format": "multiselect", - "options": [ - "Researchers", - "Third party (describe in text box below)" - ] - }, - { - "qid": "q27", - "title": "Third party implementer information", - "format": "textarea", - "required": false, - "description": "List any third party implementer(s), if third party was selected above" - }, - { - "qid": "q28", - "title": "Did any of the research team receive remuneration from the implementing agency for taking part in this research?", - "nav": "Remuneration?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A", - "Other (describe in text box below)" - ] - }, - { - "qid": "q29", - "title": "Other renumeration information", - "format": "textarea", - "required": false, - "description": "Other (additional information if 'other' was selected above)" - }, - { - "qid": "q30", - "title": "If relevant, is there an advance agreement with the implementation group that all results can be published?", - "nav": "is there an advance agreement with the implementation group that all results can be published?", - "type": "choose", - "format": "singleselect", - "options": [ - "Yes", - "No", - "N/A", - "Other (describe in text box below)" - ] - }, - { - "qid": "q31", - "title": "Other publication agreement information", - "format": "textarea", - "required": false, - "description": "Other (additional information if 'other' was selected above)" - }, - { - "qid": "q32", - "title": "JEL classification(s)", - "nav": "JEL classification(s)", - "type": "string", - "format": "text", - "description": "Please provide alphanumeric code(s). If multiple classifications, separate by commas (e.g. D31, C19, F22). For more information, see https://www.aeaweb.org/jel/guide/jel.php" - } - ] - }, - { - "id": "page3", - "title": "Keywords and Data", - "questions": [{ - "qid": "q33", - "nav": "Keywords", - "type": "choose", - "format": "multiselect", - "title": "Keywords for Methodology", - "description": "Choose one or more categories that describe your study methodology.", - "options": [ - "Experimental Design", - "Field Experiments", - "Lab Experiments", - "Mixed Method", - "Statistics", - "Survey Methodology" - ] - }, { - "qid": "q34", - "nav": "Keywords", - "type": "choose", - "format": "multiselect", - "title": "Keywords for Policy", - "description": "Choose one or more policy categories.", - "options": [ - "Conflict and Violence", - "Corruption", - "Development", - "Elections", - "Ethnic Politics", - "Gender", - "Governance" - ] - }, { - "qid": "q35", - "title": "Certification", - "nav": "Certification", - "type": "choose", - "format": "singleselect", - "description": "By submitting this form and accompanying documents with EGAP, I confirm that I have rights to put this information in the public domain and I understand that this information will remain on the EGAP registry in perpetuity, regardless of whether the research is subsequently implemented or not.", - "options": [ - "Agree" - ], - "required": true - }, { - "qid": "q36", - "title": "Confirmation", - "nav": "Confirmation", - "type": "choose", - "format": "singleselect", - "description": "You should receive a confirmation of your registration within three business days. Your registration is considered complete only when confirmation is received. If you do not receive confirmation within five business days please contact paps@egap.org.", - "options": [ - "Agree" - ], - "required": true - }, { - "qid": "q37", - "nav": "Additional documentation", - "title": "Additional documentation", - "type": "osf-upload", - "format": "osf-upload-open", - "description": "Please upload your pre-analysis plan, along with any other supporting documents, such as survey instrument, research protocol, any data, etc. OSF functionality allows for de-identified registration links to be generated after the registration is complete. However, it cannot automatically update the documents you attach.If you plan to generate a de-identified link for this registration (for journal submission/review purposes, for instance), please MAKE SURE ALL FILES UPLOADED HERE ARE ANONYMIZED" - }, { - "qid": "q38", - "nav": "DeclareDesign", - "title": "DeclareDesign", - "type": "osf-upload", - "format": "osf-upload-open", - "description": "If you have used DeclareDesign to simulate your study, attach the output here. DeclareDesign is a system for describing research designs in code and simulating them in order to understand their properties. Learn more at https://declaredesign.org/." - }] - } - ] -} diff --git a/scripts/EGAP/egap-registration.json b/scripts/EGAP/egap-registration.json deleted file mode 100644 index 28d3721e8bf2..000000000000 --- a/scripts/EGAP/egap-registration.json +++ /dev/null @@ -1,382 +0,0 @@ -{ - "name": "EGAP Registration", - "version": 2, - "description": "The EGAP registry focuses on designs for experiments and observational studies in governance and politics.", - "pages": [{ - "id": "page1", - "title": "General Information About the Project", - "questions": [{ - "qid": "q1", - "nav": "Title", - "type": "string", - "format": "text", - "title": "B1 Title of Study", - "description": "Provide the working title of your study.", - "required": true - }, - { - "qid": "q2", - "nav": "Authors", - "title": "B2 Authors", - "help": "Jimmy Stewart, Ava Gardner, Bob Hope, Greta Garbo", - "format": "textarea", - "required": true - }, - { - "qid": "q3", - "nav": "EGAP Registration ID", - "title": "EGAP Registration ID", - "format": "textarea", - "required": true - }, - { - "qid": "q4", - "nav": "Timestamp", - "title": "Timestamp of original registration", - "format": "textarea", - "required": true - }, - { - "qid": "q5", - "nav": "Acknowledgements", - "title": "B3 Acknowledgements", - "type": "string", - "format": "textarea", - "required": false - }, - { - "qid": "q6", - "title": "B4 Is one of the study authors a university faculty member?", - "nav": "University Faculty Member?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "Yes", - "No", - "Other (describe in text box below)" - ], - "description": "Please choose one" - }, - { - "qid": "q7", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q8", - "title": "B5 Is this Registration Prospective or Retrospective?", - "nav": "Prospective or Retrospective?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "Registration prior to any research activities", - "Registration prior to assignment of treatment", - "Registration prior to realization of outcomes", - "Registration prior to researcher access to outcome data", - "Registration prior to researcher analysis of outcome data", - "Registration after researcher analysis of outcome data", - "Other (describe in text box below)" - ], - "description": "Please choose one" - }, - { - "qid": "q9", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q10", - "title": "B6 Is this an experimental study?", - "nav": "Experimental study?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "Yes", - "No" - ], - "description": "(with random assignment of units to different conditions)" - }, - { - "qid": "q11", - "title": "B7 Date of start of study", - "nav": "Date of start of study", - "type": "string", - "format": "text", - "description": "Understood as first date of treatment assignment or equivalent for observational study", - "help": "E.g., 06/02/2018" - }, - { - "qid": "q12", - "title": "B8 Gate Date", - "nav": "Gate Date?", - "type": "string", - "format": "text", - "description": "Gating is discouraged, but if necessary, EGAP policy limits the gate range to 18 months maximum.", - "help": "E.g., 06/02/2018" - }, - { - "qid": "q13", - "title": "B9 Was this design presented at an EGAP meeting?", - "nav": "Presented at an EGAP meeting?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "No", - "Yes" - ], - "description": "Indicate if the design received feedback from a EGAP design workshop or other special EGAP session prior to registration" - }, - { - "qid": "q14", - "title": "B10 Is there a pre-analysis plan associated with this registration?", - "nav": "Pre-analysis plan associated with this registration?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "No", - "Yes" - ], - "description": "If so, please attach it in the Additional Documentation section on the final screen." - } - ] - }, - { - "id": "page2", - "title": "Registration Data", - "questions": [{ - "qid": "q15", - "nav": "Background and explanation of rationale.", - "title": "C1 Background and explanation of rationale.", - "format": "textarea", - "required": true, - "description": "Brief description of goals of project. If you are also attaching a pre-analysis plan, please refrain from simply copying and pasting a section from your plan here. If possible, please also avoid saying \"see attached pre-analysis plan,\" as it renders the search functionality less useful. Rather, please provide a short (1-2 paragraph) summary of the project background." - }, - { - "qid": "q16", - "nav": "Background and explanation of rationale.", - "title": "C2 What are the hypotheses to be tested/quantities of interest to be estimated?", - "format": "textarea", - "required": true, - "description": "Please list the hypotheses including hypotheses on heterogeneous effects. If you are also attaching a pre-analysis plan, please refrain from simply copying and pasting a section from your plan here. If possible, please also avoid saying \"see attached pre-analysis plan,\" as it renders the search functionality less useful. Rather, please provide a short (1-2 paragraph) summary of project hypotheses." - }, - { - "qid": "q17", - "nav": "How will these hypotheses be tested?", - "title": "C3 How will these hypotheses be tested?", - "format": "textarea", - "required": true, - "description": "Brief description of your methodology. If you are also attaching a pre-analysis plan, please refrain from simply copying and pasting a section from your plan here. If possible, please also avoid saying \"see attached pre-analysis plan,\" as it renders the search functionality less useful. Rather, please provide a short (1-2 paragraph) summary of project methodology." - }, - { - "qid": "q18", - "title": "C4 Country", - "nav": "Country", - "type": "string", - "format": "text", - "help": "comma separated names of countries (e.g. Canada, United States of America, Mexico)" - }, - { - "qid": "q19", - "title": "C5 Sample Size (# of Units)", - "nav": "Sample Size", - "type": "string", - "format": "text" - }, - { - "qid": "q20", - "title": "C6 Was a power analysis conducted prior to data collection?", - "nav": "Power analysis conducted prior to data collection?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "No", - "Yes", - "Other (describe in text box below)" - ] - }, - { - "qid": "q21", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q22", - "title": "C7 Has this research received Institutional Review Board (IRB) or ethics committee approval?", - "nav": "Review Board (IRB) or ethics committee approval?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "No", - "Yes", - "Other (describe in text box below)" - ] - }, - { - "qid": "q23", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q24", - "title": "C8 IRB Number", - "nav": "IRB Number", - "type": "string", - "format": "text" - }, - { - "qid": "q25", - "title": "C9 Date of IRB Approval", - "nav": "IRB Number", - "type": "string", - "format": "text" - }, - { - "qid": "q26", - "title": "C10 Will the intervention be implemented by the researcher or a third party? If a third party, please provide the name.", - "nav": "Review Board (IRB) or ethics committee approval?", - "type": "choose", - "format": "singleselect", - "options": [ - "Researchers", - "Other (describe in text box below)" - ] - }, - { - "qid": "q27", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q28", - "title": "C11 Did any of the research team receive remuneration from the implementing agency for taking part in this research?", - "nav": "Remuneration?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "Yes", - "No", - "Other (describe in text box below)" - ] - }, - { - "qid": "q29", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q30", - "title": "C12 If relevant, is there an advance agreement with the implementation group that all results can be published?", - "nav": "is there an advance agreement with the implementation group that all results can be published?", - "type": "choose", - "format": "singleselect", - "options": [ - "N/A", - "Yes", - "No", - "Other (describe in text box below)" - ] - }, - { - "qid": "q31", - "title": "Other", - "format": "textarea", - "required": false - }, - { - "qid": "q32", - "title": "C13 JEL classification(s)", - "nav": "JEL classification(s)", - "type": "string", - "format": "text", - "description": "Please provide alphanumeric code(s). If multiple classifications, separate by commas (e.g. D31, C19, F22)" - } - ] - }, - { - "id": "page3", - "title": "Keywords and Data", - "questions": [{ - "qid": "q33", - "nav": "Keywords", - "type": "choose", - "format": "multiselect", - "title": "Keywords for Methodology", - "description": "Choose one or more categories that describe your study methodology.", - "options": [ - "Experimental Design", - "Field Experiments", - "Lab Experiments", - "Mixed Method", - "Statistics", - "Survey Methodology" - ] - }, { - "qid": "q34", - "nav": "Keywords", - "type": "choose", - "format": "multiselect", - "title": "Keywords for Policy", - "description": "Choose one or more policy categories.", - "options": [ - "Conflict and Violence", - "Corruption", - "Development", - "Elections", - "Ethnic Politics", - "Gender", - "Governance" - ] - }, { - "qid": "q35", - "title": "Certification", - "nav": "Certification", - "type": "choose", - "format": "singleselect", - "description": "By submitting this form and accompanying documents with EGAP, I confirm that I have rights to put this information in the public domain and I understand that this information will remain on the EGAP registry in perpetuity, regardless of whether the research is subsequently implemented or not.", - "options": [ - "Agree" - ], - "required": true - }, { - "qid": "q36", - "title": "Confirmation", - "nav": "Confirmation", - "type": "choose", - "format": "singleselect", - "description": "You should receive a confirmation of your registration within three business days. Your registration is considered complete only when confirmation is received. If you do not receive confirmation within three business days please contact paps@egap.org.", - "options": [ - "Agree" - ], - "required": true - }, { - "qid": "q37", - "nav": "Additional Documentation", - "title": "Additional Documentation", - "type": "osf-upload", - "format": "osf-upload-open", - "description": "Please upload your pre-analysis plan, along with any other supporting documents, such as survey instrument, research protocol, any data, etc." - }, { - "qid": "q38", - "nav": "Anonymous Documentation", - "title": "Anonymous Documentation", - "type": "osf-upload", - "format": "osf-upload-open", - "description": "Please upload your anonymized pre-analysis plan, along with any other supporting documents, such as survey instrument, research protocol, any data, etc." - }] - } - ] -} diff --git a/scripts/EGAP/egap_workflow.ipynb b/scripts/EGAP/egap_workflow.ipynb deleted file mode 100644 index 96f5af427038..000000000000 --- a/scripts/EGAP/egap_workflow.ipynb +++ /dev/null @@ -1,1542 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "From the `scripts/EGAP` folder, with your virtualenv active, `pip install -r requirements.txt`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from create_EGAP_json import create_file_tree_and_json\n", - "from files_to_import_structure import audit_files, main as convert_files\n", - "\n", - "author_source = '/Users/bgeiger/Desktop/EGAP/20190821_author_emails.csv'\n", - "registry_source = '/Users/bgeiger/Desktop/EGAP/20191014_OSF_database.csv'\n", - "metadata_directory = '/Users/bgeiger/Desktop/EGAP/metadata/'\n", - "raw_files_directory = '/Users/bgeiger/Desktop/EGAP/raw_files/'\n", - "directories_to_process = [\n", - " '20110302AA',\n", - " '20110307AA',\n", - " '20120117AA',\n", - " '20120220AA',\n", - " '20120727AA',\n", - " '20120925AA',\n", - " '20120926AA',\n", - " '20121001AA',\n", - " '20121002AA',\n", - " '20121012AA',\n", - " '20121026AA',\n", - " '20121031AA',\n", - " '20121101AA',\n", - " '20121104AA',\n", - " '20121106AA',\n", - " '20121107AA',\n", - " '20121123AA',\n", - " '20121212AA',\n", - " '20130122AA',\n", - " '20130403AA',\n", - " '20130406AA',\n", - " '20130410AA',\n", - " '20130426AA',\n", - " '20130518AA',\n", - " '20130607AA',\n", - " '20130616AA',\n", - " '20130704AA',\n", - " '20130729AA',\n", - " '20130731AA',\n", - " '20130803AA',\n", - " '20130813AA',\n", - " '20130819AA',\n", - " '20130913AA',\n", - " '20130921AA',\n", - " '20131012AA',\n", - " '20131024AA',\n", - " '20131101AA',\n", - " '20131105AA',\n", - " '20131110AA',\n", - " '20131117AA',\n", - " '20131118AA',\n", - " '20131130AA',\n", - " '20131203AA',\n", - " '20131206AA',\n", - " '20131210AA',\n", - " '20131211AA',\n", - " '20131216AA',\n", - " '20131220AA',\n", - " '20140110AA',\n", - " '20140112AA',\n", - " '20140113AA',\n", - " '20140120AA',\n", - " '20140124AA',\n", - " '20140126AA',\n", - " '20140131AA',\n", - " '20140203AA',\n", - " '20140203AB',\n", - " '20140222AA',\n", - " '20140222AB',\n", - " '20140228AA',\n", - " '20140303AA',\n", - " '20140308AA',\n", - " '20140316AA',\n", - " '20140320AA',\n", - " '20140417AA',\n", - " '20140502AA',\n", - " '20140503AA',\n", - " '20140506AA',\n", - " '20140509AA',\n", - " '20140509AB',\n", - " '20140512AA',\n", - " '20140521AA',\n", - " '20140523AA',\n", - " '20140529AA',\n", - " '20140610AA',\n", - " '20140611AA',\n", - " '20140611AB',\n", - " '20140613AB',\n", - " '20140627AA',\n", - " '20140627AB',\n", - " '20140627AC',\n", - " '20140701AA',\n", - " '20140701AB',\n", - " '20140707AA',\n", - " '20140708AA',\n", - " '20140715AA',\n", - " '20140722AA',\n", - " '20140723AA',\n", - " '20140723AB',\n", - " '20140806AA',\n", - " '20140812AA',\n", - " '20140820AA',\n", - " '20140912AA',\n", - " '20140915AA',\n", - " '20140918AA',\n", - " '20140922AA',\n", - " '20141002AA',\n", - " '20141006AA',\n", - " '20141023AA',\n", - " '20141025AA',\n", - " '20141027AA',\n", - " '20141031AA',\n", - " '20141031AB',\n", - " '20141101AA',\n", - " '20141103AA',\n", - " '20141107AA',\n", - " '20141117AA',\n", - " '20141202AA',\n", - " '20141208AA',\n", - " '20141213AA',\n", - " '20141223AA',\n", - " '20141225AA',\n", - " '20141227AA',\n", - " '20141231AA',\n", - " '20150110AA',\n", - " '20150111AA',\n", - " '20150118AA',\n", - " '20150122AA',\n", - " '20150122AB',\n", - " '20150127AA',\n", - " '20150131AA',\n", - " '20150202AA',\n", - " '20150204AA',\n", - " '20150206AA',\n", - " '20150211AA',\n", - " '20150216AA',\n", - " '20150304AA',\n", - " '20150308AA',\n", - " '20150309AA',\n", - " '20150310AA',\n", - " '20150311AA',\n", - " '20150313AA',\n", - " '20150320AA',\n", - " '20150323AA',\n", - " '20150324AA',\n", - " '20150326AA',\n", - " '20150330AA',\n", - " '20150420AA',\n", - " '20150423AA',\n", - " '20150428AA',\n", - " '20150429AA',\n", - " '20150508AA',\n", - " '20150513AA',\n", - " '20150513AB',\n", - " '20150513AC',\n", - " '20150513AD',\n", - " '20150513AE',\n", - " '20150513AF',\n", - " '20150513AG',\n", - " '20150513AH',\n", - " '20150513AI',\n", - " '20150514AA',\n", - " '20150517AA',\n", - " '20150518AA',\n", - " '20150520AA',\n", - " '20150522AA',\n", - " '20150526AA',\n", - " '20150527AA',\n", - " '20150602AA',\n", - " '20150602AB',\n", - " '20150603AA',\n", - " '20150604AA',\n", - " '20150605AA',\n", - " '20150605AB',\n", - " '20150616AA',\n", - " '20150617AA',\n", - " '20150619AA',\n", - " '20150622AA',\n", - " '20150623AA',\n", - " '20150701AA',\n", - " '20150702AA',\n", - " '20150703AA',\n", - " '20150707AA',\n", - " '20150708AA',\n", - " '20150709AA',\n", - " '20150709AB',\n", - " '20150710AA',\n", - " '20150713AA',\n", - " '20150716AA',\n", - " '20150716AB',\n", - " '20150717AA',\n", - " '20150718AA',\n", - " '20150720AA',\n", - " '20150723AA',\n", - " '20150724AA',\n", - " '20150727AA',\n", - " '20150731AA',\n", - " '20150731AB',\n", - " '20150803AA',\n", - " '20150803AB',\n", - " '20150812AA',\n", - " '20150813AA',\n", - " '20150813AB',\n", - " '20150819AA',\n", - " '20150819AB',\n", - " '20150820AA',\n", - " '20150824AA',\n", - " '20150824AB',\n", - " '20150825AA',\n", - " '20150827AA',\n", - " '20150903AA',\n", - " '20150903AB',\n", - " '20150914AA',\n", - " '20150915AA',\n", - " '20150917AA',\n", - " '20150921AA',\n", - " '20150922AA',\n", - " '20150924AA',\n", - " '20150925AA',\n", - " '20150927AA',\n", - " '20150928AA',\n", - " '20150928AB',\n", - " '20150929AA',\n", - " '20150929AB',\n", - " '20150930AA',\n", - " '20150930AB',\n", - " '20151003AA',\n", - " '20151006AA',\n", - " '20151006AB',\n", - " '20151012AA',\n", - " '20151013AA',\n", - " '20151013AB',\n", - " '20151014AA',\n", - " '20151014AB',\n", - " '20151016AA',\n", - " '20151016AB',\n", - " '20151016AC',\n", - " '20151017AA',\n", - " '20151019AA',\n", - " '20151023AA',\n", - " '20151027AA',\n", - " '20151030AA',\n", - " '20151102AA',\n", - " '20151102AB',\n", - " '20151102AC',\n", - " '20151103AA',\n", - " '20151107AA',\n", - " '20151112AA',\n", - " '20151112AB',\n", - " '20151114AA',\n", - " '20151116AA',\n", - " '20151116AB',\n", - " '20151118AA',\n", - " '20151119AA',\n", - " '20151119AB',\n", - " '20151120AA',\n", - " '20151120AB',\n", - " '20151123AA',\n", - " '20151125AA',\n", - " '20151125AB',\n", - " '20151128AA',\n", - " '20151201AA',\n", - " '20151201AB',\n", - " '20151202AA',\n", - " '20151204AA',\n", - " '20151206AA',\n", - " '20151207AA',\n", - " '20151209AA',\n", - " '20151218AA',\n", - " '20160105AA',\n", - " '20160106AA',\n", - " '20160112AA',\n", - " '20160112AB',\n", - " '20160113AA',\n", - " '20160113AB',\n", - " '20160119AA',\n", - " '20160121AA',\n", - " '20160202AA',\n", - " '20160208AA',\n", - " '20160208AB',\n", - " '20160208AC',\n", - " '20160216AA',\n", - " '20160217AA',\n", - " '20160219AA',\n", - " '20160222AA',\n", - " '20160224AA',\n", - " '20160224AB',\n", - " '20160225AA',\n", - " '20160308AA',\n", - " '20160308AB',\n", - " '20160309AA',\n", - " '20160313AA',\n", - " '20160315AA',\n", - " '20160318AA',\n", - " '20160321AA',\n", - " '20160323AA',\n", - " '20160324AA',\n", - " '20160327AA',\n", - " '20160330AA',\n", - " '20160401AA',\n", - " '20160404AA',\n", - " '20160404AB',\n", - " '20160405AA',\n", - " '20160405AB',\n", - " '20160406AA',\n", - " '20160408AA',\n", - " '20160409AA',\n", - " '20160409AB',\n", - " '20160410AA',\n", - " '20160411AA',\n", - " '20160411AB',\n", - " '20160411AC',\n", - " '20160411AD',\n", - " '20160413AA',\n", - " '20160414AA',\n", - " '20160415AA',\n", - " '20160416AA',\n", - " '20160416AB',\n", - " '20160421AA',\n", - " '20160426AA',\n", - " '20160427AA',\n", - " '20160427AB',\n", - " '20160429AA',\n", - " '20160429AB',\n", - " '20160507AA',\n", - " '20160514AA',\n", - " '20160515AA',\n", - " '20160516AA',\n", - " '20160517AA',\n", - " '20160517AB',\n", - " '20160517AC',\n", - " '20160517AD',\n", - " '20160517AE',\n", - " '20160519AA',\n", - " '20160520AA',\n", - " '20160524AA',\n", - " '20160531AA',\n", - " '20160601AA',\n", - " '20160601AB',\n", - " '20160601AC',\n", - " '20160601AD',\n", - " '20160602AA',\n", - " '20160603AA',\n", - " '20160605AA',\n", - " '20160607AA',\n", - " '20160607AB',\n", - " '20160609AA',\n", - " '20160609AB',\n", - " '20160609AC',\n", - " '20160611AA',\n", - " '20160612AA',\n", - " '20160613AA',\n", - " '20160613AB',\n", - " '20160615AA',\n", - " '20160615AB',\n", - " '20160616AA',\n", - " '20160617AA',\n", - " '20160617AB',\n", - " '20160618AA',\n", - " '20160621AA',\n", - " '20160621AB',\n", - " '20160621AC',\n", - " '20160621AD',\n", - " '20160622AA',\n", - " '20160622AB',\n", - " '20160624AA',\n", - " '20160624AB',\n", - " '20160625AA',\n", - " '20160628AA',\n", - " '20160629AA',\n", - " '20160630AA',\n", - " '20160702AA',\n", - " '20160708AA',\n", - " '20160708AB',\n", - " '20160712AA',\n", - " '20160717AA',\n", - " '20160719AA',\n", - " '20160719AB',\n", - " '20160721AA',\n", - " '20160726AA',\n", - " '20160726AB',\n", - " '20160727AA',\n", - " '20160729AA',\n", - " '20160730AA',\n", - " '20160801AA',\n", - " '20160801AB',\n", - " '20160803AA',\n", - " '20160803AB',\n", - " '20160809AA',\n", - " '20160809AB',\n", - " '20160812AA',\n", - " '20160812AB',\n", - " '20160813AA',\n", - " '20160813AB',\n", - " '20160813AC',\n", - " '20160819AA',\n", - " '20160819AB',\n", - " '20160820AA',\n", - " '20160822AA',\n", - " '20160824AA',\n", - " '20160825AA',\n", - " '20160829AA',\n", - " '20160831AA',\n", - " '20160905AA',\n", - " '20160907AA',\n", - " '20160911AA',\n", - " '20160912AB',\n", - " '20160913AA',\n", - " '20160913AB',\n", - " '20160916AA',\n", - " '20160916AB',\n", - " '20160918AA',\n", - " '20160919AA',\n", - " '20160921AA',\n", - " '20160921AB',\n", - " '20160926AA',\n", - " '20160926AB',\n", - " '20160926AC',\n", - " '20161001AA',\n", - " '20161004AA',\n", - " '20161006AA',\n", - " '20161011AA',\n", - " '20161016AA',\n", - " '20161017AA',\n", - " '20161019AA',\n", - " '20161019AB',\n", - " '20161020AA',\n", - " '20161026AA',\n", - " '20161028AA',\n", - " '20161030AA',\n", - " '20161101AA',\n", - " '20161101AB',\n", - " '20161103AA',\n", - " '20161103AB',\n", - " '20161104AA',\n", - " '20161104AB',\n", - " '20161109AA',\n", - " '20161109AB',\n", - " '20161109AC',\n", - " '20161110AA',\n", - " '20161110AB',\n", - " '20161110AC',\n", - " '20161110AD',\n", - " '20161110AE',\n", - " '20161112AA',\n", - " '20161118AA',\n", - " '20161121AA',\n", - " '20161122AA',\n", - " '20161122AB',\n", - " '20161123AA',\n", - " '20161125AA',\n", - " '20161127AA',\n", - " '20161128AA',\n", - " '20161129AA',\n", - " '20161204AA',\n", - " '20161204AB',\n", - " '20161205AA',\n", - " '20161206AA',\n", - " '20161207AA',\n", - " '20161208AA',\n", - " '20161212AA',\n", - " '20161216AA',\n", - " '20161216AB',\n", - " '20161227AA',\n", - " '20161227AB',\n", - " '20170103AA',\n", - " '20170109AA',\n", - " '20170112AA',\n", - " '20170115AA',\n", - " '20170117AA',\n", - " '20170118AA',\n", - " '20170123AA',\n", - " '20170124AA',\n", - " '20170130AA',\n", - " '20170131AA',\n", - " '20170203AA',\n", - " '20170203AB',\n", - " '20170203AC',\n", - " '20170203AD',\n", - " '20170205AA',\n", - " '20170207AA',\n", - " '20170208AA',\n", - " '20170209AA',\n", - " '20170210AA',\n", - " '20170212AA',\n", - " '20170214AA',\n", - " '20170214AB',\n", - " '20170215AA',\n", - " '20170216AA',\n", - " '20170216AB',\n", - " '20170220AA',\n", - " '20170222AA',\n", - " '20170223AA',\n", - " '20170223AB',\n", - " '20170223AC',\n", - " '20170224AA',\n", - " '20170225AA',\n", - " '20170227AA',\n", - " '20170227AB',\n", - " '20170227AC',\n", - " '20170301AA',\n", - " '20170302AA',\n", - " '20170307AA',\n", - " '20170308AA',\n", - " '20170308AB',\n", - " '20170309AA',\n", - " '20170310AA',\n", - " '20170312AA',\n", - " '20170317AA',\n", - " '20170320AA',\n", - " '20170320AB',\n", - " '20170320AC',\n", - " '20170321AA',\n", - " '20170321AB',\n", - " '20170322AA',\n", - " '20170322AB',\n", - " '20170323AA',\n", - " '20170324AA',\n", - " '20170325AA',\n", - " '20170328AA',\n", - " '20170329AA',\n", - " '20170330AA',\n", - " '20170403AA',\n", - " '20170412AA',\n", - " '20170412AB',\n", - " '20170413AA',\n", - " '20170413AB',\n", - " '20170413AC',\n", - " '20170414AA',\n", - " '20170416AA',\n", - " '20170417AA',\n", - " '20170417AB',\n", - " '20170420AA',\n", - " '20170421AA',\n", - " '20170422AA',\n", - " '20170423AA',\n", - " '20170423AB',\n", - " '20170426AA',\n", - " '20170427AA',\n", - " '20170428AA',\n", - " '20170501AA',\n", - " '20170501AB',\n", - " '20170501AC',\n", - " '20170501AD',\n", - " '20170503AA',\n", - " '20170503AB',\n", - " '20170503AC',\n", - " '20170504AA',\n", - " '20170504AB',\n", - " '20170505AA',\n", - " '20170505AB',\n", - " '20170505AC',\n", - " '20170506AA',\n", - " '20170507AA',\n", - " '20170508AA',\n", - " '20170508AB',\n", - " '20170509AA',\n", - " '20170509AB',\n", - " '20170510AA',\n", - " '20170511AA',\n", - " '20170515AA',\n", - " '20170515AB',\n", - " '20170516AA',\n", - " '20170517AA',\n", - " '20170519AA',\n", - " '20170520AA',\n", - " '20170522AA',\n", - " '20170523AA',\n", - " '20170524AA',\n", - " '20170525AA',\n", - " '20170525AB',\n", - " '20170527AA',\n", - " '20170531AA',\n", - " '20170602AA',\n", - " '20170603AA',\n", - " '20170606AA',\n", - " '20170608AA',\n", - " '20170609AA',\n", - " '20170609AB',\n", - " '20170609AC',\n", - " '20170611AA',\n", - " '20170611AB',\n", - " '20170612AA',\n", - " '20170613AA',\n", - " '20170614AA',\n", - " '20170615AA',\n", - " '20170615AB',\n", - " '20170616AA',\n", - " '20170617AA',\n", - " '20170618AA',\n", - " '20170619AA',\n", - " '20170626AA',\n", - " '20170626AB',\n", - " '20170629AA',\n", - " '20170705AA',\n", - " '20170706AA',\n", - " '20170706AB',\n", - " '20170706AC',\n", - " '20170711AA',\n", - " '20170712AA',\n", - " '20170714AA',\n", - " '20170716AA',\n", - " '20170716AB',\n", - " '20170717AA',\n", - " '20170720AA',\n", - " '20170720AB',\n", - " '20170720AC',\n", - " '20170721AA',\n", - " '20170721AB',\n", - " '20170724AA',\n", - " '20170725AA',\n", - " '20170725AB',\n", - " '20170727AA',\n", - " '20170728AA',\n", - " '20170728AB',\n", - " '20170729AA',\n", - " '20170731AA',\n", - " '20170803AA',\n", - " '20170804AA',\n", - " '20170805AA',\n", - " '20170807AA',\n", - " '20170808AA',\n", - " '20170809AA',\n", - " '20170810AA',\n", - " '20170811AA',\n", - " '20170811AB',\n", - " '20170814AA',\n", - " '20170815AA',\n", - " '20170816AA',\n", - " '20170819AA',\n", - " '20170821AA',\n", - " '20170821AB',\n", - " '20170822AA',\n", - " '20170823AA',\n", - " '20170828AA',\n", - " '20170829AA',\n", - " '20170831AA',\n", - " '20170901AA',\n", - " '20170905AA',\n", - " '20170906AA',\n", - " '20170907AA',\n", - " '20170908AA',\n", - " '20170908AB',\n", - " '20170908AC',\n", - " '20170910AA',\n", - " '20170911AA',\n", - " '20170913AA',\n", - " '20170913AB',\n", - " '20170913AC',\n", - " '20170914AA',\n", - " '20170914AB',\n", - " '20170915AA',\n", - " '20170915AB',\n", - " '20170918AA',\n", - " '20170919AA',\n", - " '20170920AA',\n", - " '20170920AB',\n", - " '20170920AC',\n", - " '20170921AA',\n", - " '20170922AA',\n", - " '20170922AB',\n", - " '20170922AC',\n", - " '20170925AA',\n", - " '20170926AA',\n", - " '20170926AB',\n", - " '20170927AA',\n", - " '20170927AB',\n", - " '20170928AA',\n", - " '20170929AA',\n", - " '20170930AA',\n", - " '20171001AA',\n", - " '20171001AB',\n", - " '20171002AA',\n", - " '20171003AA',\n", - " '20171003AB',\n", - " '20171004AA',\n", - " '20171009AA',\n", - " '20171010AA',\n", - " '20171010AB',\n", - " '20171012AA',\n", - " '20171013AA',\n", - " '20171015AA',\n", - " '20171016AA',\n", - " '20171017AA',\n", - " '20171018AA',\n", - " '20171019AA',\n", - " '20171020AA',\n", - " '20171022AA',\n", - " '20171023AA',\n", - " '20171024AA',\n", - " '20171024AB',\n", - " '20171024AC',\n", - " '20171025AA',\n", - " '20171027AA',\n", - " '20171101AA',\n", - " '20171103AA',\n", - " '20171104AA',\n", - " '20171104AB',\n", - " '20171105AA',\n", - " '20171106AA',\n", - " '20171106AB',\n", - " '20171106AC',\n", - " '20171107AA',\n", - " '20171109AA',\n", - " '20171109AB',\n", - " '20171113AA',\n", - " '20171113AB',\n", - " '20171113AC',\n", - " '20171114AA',\n", - " '20171115AA',\n", - " '20171117AA',\n", - " '20171117AB',\n", - " '20171117AC',\n", - " '20171119AA',\n", - " '20171120AA',\n", - " '20171120AB',\n", - " '20171121AA',\n", - " '20171121AB',\n", - " '20171122AA',\n", - " '20171122AB',\n", - " '20171122AC',\n", - " '20171124AA',\n", - " '20171127AA',\n", - " '20171127AB',\n", - " '20171128AA',\n", - " '20171129AA',\n", - " '20171205AA',\n", - " '20171205AB',\n", - " '20171206AA',\n", - " '20171208AA',\n", - " '20171210AA',\n", - " '20171210AB',\n", - " '20171211AA',\n", - " '20171211AB',\n", - " '20171211AC',\n", - " '20171213AA',\n", - " '20171218AA',\n", - " '20171218AB',\n", - " '20171218AC',\n", - " '20171221AA',\n", - " '20171222AA',\n", - " '20171223AA',\n", - " '20171228AA',\n", - " '20171229AA',\n", - " '20171230AA',\n", - " '20180105AA',\n", - " '20180105AB',\n", - " '20180105AC',\n", - " '20180105AD',\n", - " '20180108AA',\n", - " '20180108AB',\n", - " '20180109AA',\n", - " '20180109AB',\n", - " '20180110AA',\n", - " '20180110AB',\n", - " '20180113AA',\n", - " '20180119AA',\n", - " '20180120AA',\n", - " '20180121AA',\n", - " '20180123AA',\n", - " '20180124AA',\n", - " '20180125AA',\n", - " '20180126AA',\n", - " '20180126AB',\n", - " '20180127AA',\n", - " '20180128AA',\n", - " '20180130AA',\n", - " '20180201AA',\n", - " '20180201AB',\n", - " '20180201AC',\n", - " '20180202AA',\n", - " '20180202AB',\n", - " '20180202AC',\n", - " '20180204AA',\n", - " '20180204AB',\n", - " '20180205AA',\n", - " '20180205AB',\n", - " '20180205AC',\n", - " '20180206AA',\n", - " '20180208AA',\n", - " '20180208AB',\n", - " '20180209AA',\n", - " '20180211AA',\n", - " '20180213AA',\n", - " '20180213AB',\n", - " '20180214AA',\n", - " '20180215AA',\n", - " '20180215AB',\n", - " '20180215AC',\n", - " '20180219AA',\n", - " '20180219AB',\n", - " '20180220AA',\n", - " '20180221AA',\n", - " '20180221AB',\n", - " '20180222AA',\n", - " '20180222AB',\n", - " '20180227AA',\n", - " '20180228AA',\n", - " '20180228AB',\n", - " '20180302AA',\n", - " '20180303AA',\n", - " '20180304AA',\n", - " '20180304AB',\n", - " '20180304AC',\n", - " '20180304AD',\n", - " '20180305AA',\n", - " '20180306AA',\n", - " '20180308AA',\n", - " '20180310AA',\n", - " '20180313AA',\n", - " '20180315AA',\n", - " '20180315AB',\n", - " '20180315AC',\n", - " '20180316AA',\n", - " '20180316AB',\n", - " '20180318AA',\n", - " '20180319AA',\n", - " '20180319AB',\n", - " '20180319AC',\n", - " '20180320AA',\n", - " '20180321AA',\n", - " '20180323AA',\n", - " '20180323AB',\n", - " '20180324AA',\n", - " '20180325AA',\n", - " '20180327AA',\n", - " '20180328AA',\n", - " '20180329AA',\n", - " '20180331AA',\n", - " '20180401AA',\n", - " '20180402AA',\n", - " '20180402AB',\n", - " '20180403AA',\n", - " '20180404AA',\n", - " '20180409AA',\n", - " '20180409AB',\n", - " '20180409AC',\n", - " '20180413AA',\n", - " '20180413AB',\n", - " '20180416AA',\n", - " '20180417AA',\n", - " '20180418AA',\n", - " '20180418AB',\n", - " '20180423AA',\n", - " '20180424AA',\n", - " '20180425AA',\n", - " '20180425AB',\n", - " '20180425AC',\n", - " '20180425AD',\n", - " '20180426AA',\n", - " '20180426AB',\n", - " '20180426AC',\n", - " '20180427AA',\n", - " '20180430AA',\n", - " '20180430AB',\n", - " '20180430AC',\n", - " '20180502AA',\n", - " '20180503AA',\n", - " '20180503AB',\n", - " '20180504AA',\n", - " '20180507AA',\n", - " '20180508AA',\n", - " '20180509AA',\n", - " '20180509AB',\n", - " '20180514AA',\n", - " '20180515AA',\n", - " '20180515AB',\n", - " '20180516AA',\n", - " '20180516AB',\n", - " '20180518AA',\n", - " '20180521AA',\n", - " '20180521AB',\n", - " '20180523AA',\n", - " '20180528AA',\n", - " '20180529AA',\n", - " '20180529AB',\n", - " '20180529AC',\n", - " '20180530AA',\n", - " '20180601AA',\n", - " '20180602AA',\n", - " '20180605AA',\n", - " '20180605AB',\n", - " '20180605AC',\n", - " '20180605AD',\n", - " '20180607AA',\n", - " '20180608AA',\n", - " '20180608AB',\n", - " '20180610AA',\n", - " '20180610AB',\n", - " '20180611AA',\n", - " '20180611AB',\n", - " '20180612AA',\n", - " '20180613AA',\n", - " '20180614AA',\n", - " '20180614AB',\n", - " '20180615AA',\n", - " '20180616AA',\n", - " '20180619AA',\n", - " '20180619AB',\n", - " '20180620AA',\n", - " '20180625AA',\n", - " '20180626AA',\n", - " '20180628AA',\n", - " '20180628AB',\n", - " '20180701AA',\n", - " '20180703AA',\n", - " '20180703AB',\n", - " '20180703AC',\n", - " '20180707AA',\n", - " '20180709AA',\n", - " '20180709AB',\n", - " '20180710AA',\n", - " '20180710AB',\n", - " '20180710AC',\n", - " '20180711AA',\n", - " '20180711AB',\n", - " '20180712AA',\n", - " '20180713AA',\n", - " '20180716AA',\n", - " '20180719AA',\n", - " '20180720AA',\n", - " '20180722AA',\n", - " '20180723AA',\n", - " '20180723AB',\n", - " '20180724AA',\n", - " '20180724AB',\n", - " '20180725AA',\n", - " '20180725AB',\n", - " '20180725AC',\n", - " '20180730AA',\n", - " '20180731AA',\n", - " '20180801AA',\n", - " '20180801AB',\n", - " '20180802AA',\n", - " '20180802AB',\n", - " '20180803AA',\n", - " '20180804AA',\n", - " '20180807AA',\n", - " '20180807AB',\n", - " '20180808AA',\n", - " '20180809AA',\n", - " '20180809AB',\n", - " '20180809AC',\n", - " '20180810AA',\n", - " '20180811AA',\n", - " '20180812AA',\n", - " '20180814AA',\n", - " '20180814AB',\n", - " '20180815AA',\n", - " '20180816AA',\n", - " '20180816AB',\n", - " '20180817AA',\n", - " '20180819AA',\n", - " '20180819AB',\n", - " '20180821AA',\n", - " '20180821AB',\n", - " '20180822AA',\n", - " '20180826AA',\n", - " '20180827AA',\n", - " '20180827AB',\n", - " '20180829AA',\n", - " '20180831AA',\n", - " '20180903AA',\n", - " '20180904AA',\n", - " '20180904AB',\n", - " '20180905AA',\n", - " '20180906AA',\n", - " '20180910AA',\n", - " '20180910AB',\n", - " '20180912AA',\n", - " '20180914AA',\n", - " '20180918AA',\n", - " '20180918AB',\n", - " '20180919AA',\n", - " '20180920AA',\n", - " '20180920AB',\n", - " '20180925AA',\n", - " '20180925AB',\n", - " '20180927AA',\n", - " '20180927AB',\n", - " '20181001AA',\n", - " '20181003AA',\n", - " '20181005AA',\n", - " '20181006AA',\n", - " '20181010AA',\n", - " '20181010AB',\n", - " '20181012AA',\n", - " '20181013AA',\n", - " '20181015AA',\n", - " '20181016AA',\n", - " '20181017AA',\n", - " '20181017AB',\n", - " '20181018AA',\n", - " '20181019AA',\n", - " '20181022AA',\n", - " '20181023AA',\n", - " '20181023AB',\n", - " '20181024AA',\n", - " '20181024AB',\n", - " '20181024AC',\n", - " '20181025AA',\n", - " '20181026AA',\n", - " '20181029AA',\n", - " '20181030AA',\n", - " '20181030AB',\n", - " '20181031AA',\n", - " '20181101AA',\n", - " '20181101AB',\n", - " '20181101AC',\n", - " '20181101AD',\n", - " '20181102AA',\n", - " '20181102AB',\n", - " '20181102AC',\n", - " '20181102AD',\n", - " '20181105AA',\n", - " '20181105AB',\n", - " '20181105AC',\n", - " '20181105AD',\n", - " '20181106AA',\n", - " '20181106AB',\n", - " '20181106AC',\n", - " '20181106AD',\n", - " '20181106AE',\n", - " '20181106AF',\n", - " '20181107AA',\n", - " '20181108AA',\n", - " '20181108AB',\n", - " '20181108AC',\n", - " '20181110AA',\n", - " '20181111AA',\n", - " '20181112AA',\n", - " '20181112AB',\n", - " '20181112AC',\n", - " '20181112AD',\n", - " '20181113AA',\n", - " '20181114AA',\n", - " '20181115AA',\n", - " '20181115AB',\n", - " '20181115AC',\n", - " '20181115AD',\n", - " '20181120AA',\n", - " '20181120AB',\n", - " '20181120AC',\n", - " '20181123AA',\n", - " '20181125AA',\n", - " '20181126AA',\n", - " '20181126AB',\n", - " '20181127AA',\n", - " '20181127AB',\n", - " '20181127AC',\n", - " '20181128AA',\n", - " '20181129AA',\n", - " '20181129AB',\n", - " '20181130AA',\n", - " '20181130AB',\n", - " '20181201AA',\n", - " '20181201AB',\n", - " '20181204AA',\n", - " '20181204AB',\n", - " '20181204AC',\n", - " '20181205AA',\n", - " '20181205AB',\n", - " '20181206AA',\n", - " '20181206AB',\n", - " '20181206AC',\n", - " '20181206AD',\n", - " '20181206AE',\n", - " '20181206AF',\n", - " '20181206AG',\n", - " '20181207AA',\n", - " '20181208AA',\n", - " '20181210AA',\n", - " '20181210AB',\n", - " '20181211AA',\n", - " '20181211AB',\n", - " '20181211AC',\n", - " '20181212AA',\n", - " '20181214AA',\n", - " '20181216AA',\n", - " '20181216AB',\n", - " '20181219AA',\n", - " '20181219AB',\n", - " '20181221AA',\n", - " '20181221AB',\n", - " '20181221AC',\n", - " '20181222AA',\n", - " '20181228AA',\n", - " '20181230AA',\n", - " '20190103AA',\n", - " '20190108AA',\n", - " '20190109AA',\n", - " '20190110AA',\n", - " '20190110AB',\n", - " '20190110AC',\n", - " '20190111AA',\n", - " '20190113AA',\n", - " '20190113AB',\n", - " '20190114AA',\n", - " '20190115AA',\n", - " '20190116AA',\n", - " '20190116AB',\n", - " '20190116AC',\n", - " '20190119AA',\n", - " '20190121AA',\n", - " '20190122AA',\n", - " '20190125AA',\n", - " '20190128AA',\n", - " '20190128AB',\n", - " '20190128AC',\n", - " '20190129AA',\n", - " '20190129AB',\n", - " '20190130AA',\n", - " '20190131AA',\n", - " '20190131AB',\n", - " '20190131AC',\n", - " '20190201AA',\n", - " '20190201AB',\n", - " '20190201AC',\n", - " '20190202AA',\n", - " '20190204AA',\n", - " '20190205AA',\n", - " '20190205AB',\n", - " '20190205AC',\n", - " '20190206AA',\n", - " '20190206AB',\n", - " '20190206AC',\n", - " '20190208AA',\n", - " '20190212AA',\n", - " '20190212AB',\n", - " '20190213AA',\n", - " '20190213AB',\n", - " '20190213AC',\n", - " '20190213AD',\n", - " '20190213AE',\n", - " '20190215AA',\n", - " '20190215AB',\n", - " '20190215AC',\n", - " '20190215AD',\n", - " '20190215AE',\n", - " '20190215AF',\n", - " '20190219AA',\n", - " '20190220AA',\n", - " '20190220AB',\n", - " '20190220AC',\n", - " '20190221AA',\n", - " '20190223AA',\n", - " '20190223AB',\n", - " '20190223AC',\n", - " '20190225AA',\n", - " '20190227AA',\n", - " '20190227AB',\n", - " '20190301AA',\n", - " '20190301AB',\n", - " '20190301AC',\n", - " '20190301AD',\n", - " '20190304AA',\n", - " '20190304AB',\n", - " '20190304AC',\n", - " '20190304AD',\n", - " '20190305AA',\n", - " '20190306AA',\n", - " '20190306AB',\n", - " '20190307AA',\n", - " '20190309AA',\n", - " '20190310AA',\n", - " '20190311AA',\n", - " '20190311AB',\n", - " '20190311AC',\n", - " '20190312AA',\n", - " '20190313AA',\n", - " '20190313AB',\n", - " '20190313AC',\n", - " '20190313AD',\n", - " '20190313AE',\n", - " '20190314AA',\n", - " '20190314AB',\n", - " '20190314AC',\n", - " '20190314AD',\n", - " '20190314AE',\n", - " '20190314AF',\n", - " '20190314AG',\n", - " '20190314AH',\n", - " '20190314AI',\n", - " '20190314AJ',\n", - " '20190314AK',\n", - " '20190314AL',\n", - " '20190315AA',\n", - " '20190315AB',\n", - " '20190315AC',\n", - " '20190315AD',\n", - " '20190320AA',\n", - " '20190320AB',\n", - " '20190320AC',\n", - " '20190320AD',\n", - " '20190325AA',\n", - " '20190326AA',\n", - " '20190326AB',\n", - " '20190327AA',\n", - " '20190327AB',\n", - " '20190327AC',\n", - " '20190328AA',\n", - " '20190329AA',\n", - " '20190329AB',\n", - " '20190401AA',\n", - " '20190401AB',\n", - " '20190402AA',\n", - " '20190404AA',\n", - " '20190405AA',\n", - " '20190406AA',\n", - " '20190410AA',\n", - " '20190410AB',\n", - " '20190410AC',\n", - " '20190411AA',\n", - " '20190411AB',\n", - " '20190411AC',\n", - " '20190412AA',\n", - " '20190416AA',\n", - " '20190417AA',\n", - " '20190417AB',\n", - " '20190418AA',\n", - " '20190420AA',\n", - " '20190422AA',\n", - " '20190423AA',\n", - " '20190424AA',\n", - " '20190424AB',\n", - " '20190426AA',\n", - " '20190427AA',\n", - " '20190427AB',\n", - " '20190429AA',\n", - " '20190430AA',\n", - " '20190430AB',\n", - " '20190430AC',\n", - " '20190503AA',\n", - " '20190503AB',\n", - " '20190506AA',\n", - " '20190507AA',\n", - " '20190507AB',\n", - " '20190512AA',\n", - " '20190513AA',\n", - " '20190513AB',\n", - " '20190513AC',\n", - " '20190514AA',\n", - " '20190515AA',\n", - " '20190515AB',\n", - " '20190515AC',\n", - " '20190515AD',\n", - " '20190515AE',\n", - " '20190516AA',\n", - " '20190517AA',\n", - " '20190520AA',\n", - " '20190522AA',\n", - " '20190522AB',\n", - " '20190522AC',\n", - " '20190522AD',\n", - " '20190522AE',\n", - " '20190522AF',\n", - " '20190524AA',\n", - " '20190524AB',\n", - " '20190526AA',\n", - " '20190526AB',\n", - " '20190527AA',\n", - " '20190527AB',\n", - " '20190528AA',\n", - " '20190528AB',\n", - " '20190528AC',\n", - " '20190529AA',\n", - " '20190529AB',\n", - " '20190529AC',\n", - " '20190530AA',\n", - " '20190530AB',\n", - " '20190530AC',\n", - " '20190531AA',\n", - " '20190603AA',\n", - " '20190604AA',\n", - " '20190604AB',\n", - " '20190604AC',\n", - " '20190604AD',\n", - " '20190604AE',\n", - " '20190605AA',\n", - " '20190605AB',\n", - " '20190605AC',\n", - " '20190606AA',\n", - " '20190606AB',\n", - " '20190606AC',\n", - " '20190606AD',\n", - " '20190607AA',\n", - " '20190608AA',\n", - " '20190609AA',\n", - " '20190611AA',\n", - " '20190612AA',\n", - " '20190613AA',\n", - " '20190613AB',\n", - " '20190614AA',\n", - " '20190615AA',\n", - " '20190616AA',\n", - " '20190616AB',\n", - " '20190616AC',\n", - " '20190617AA',\n", - " '20190618AA',\n", - " '20190620AA',\n", - " '20190620AB',\n", - " '20190620AC',\n", - " '20190620AD',\n", - " '20190621AA',\n", - " '20190621AB',\n", - " '20190624AA',\n", - " '20190625AA',\n", - " '20190625AB',\n", - " '20190625AC',\n", - " '20190625AD',\n", - " '20190625AE',\n", - " '20190626AA',\n", - " '20190701AA',\n", - " '20190703AA',\n", - " '20190707AA',\n", - " '20190707AB',\n", - " '20190708AA',\n", - " '20190708AB',\n", - " '20190709AA',\n", - " '20190709AB',\n", - " '20190710AA',\n", - " '20190711AA',\n", - " '20190711AB',\n", - " '20190711AC',\n", - " '20190712AA',\n", - " '20190713AA',\n", - " '20190714AA',\n", - " '20190716AA',\n", - " '20190716AB',\n", - " '20190717AA',\n", - " '20190717AB',\n", - " '20190717AC',\n", - " '20190718AA',\n", - " '20190718AB',\n", - " '20190718AC',\n", - " '20190719AA',\n", - " '20190719AB',\n", - " '20190722AA',\n", - " '20190722AB',\n", - " '20190722AC',\n", - " '20190723AA',\n", - " '20190724AA',\n", - " '20190724AB',\n", - " '20190724AC',\n", - " '20190724AD',\n", - " '20190725AA',\n", - " '20190726AA',\n", - " '20190729AA',\n", - " '20190729AB',\n", - " '20190730AA',\n", - " '20190731AA',\n", - " '20190731AB',\n", - " '20190731AC',\n", - " '20190731AD',\n", - " '20190731AE',\n", - " '20190802AA',\n", - " '20190804AA',\n", - " '20190806AA',\n", - " '20190807AA',\n", - " '20190807AB',\n", - " '20190808AA',\n", - " '20190808AB',\n", - " '20190810AA',\n", - " '20190810AB',\n", - " '20190812AA',\n", - " '20190812AB',\n", - " '20190813AA',\n", - " '20190813AB',\n", - " '20190814AA',\n", - " '20190814AB',\n", - " '20190814AC',\n", - " '20190815AA',\n", - " '20190816AA',\n", - " '20190819AA',\n", - " '20190819AB',\n", - " '20190821AA',\n", - " '20190822AA',\n", - " '20190822AB',\n", - " '20190822AC',\n", - " '20190823AA',\n", - " '20190825AA',\n", - " '20190826AA',\n", - " '20190826AB',\n", - " '20190827AA',\n", - " '20190828AA',\n", - " '20190828AB',\n", - " '20190828AC',\n", - " '20190829AA',\n", - " '20190901AA',\n", - " '20190901AB',\n", - " '20190901AC',\n", - " '20190903AA',\n", - " '20190903AB',\n", - " '20190904AA',\n", - " '20190905AA',\n", - " '20190905AB',\n", - " '20190906AA',\n", - " '20190906AB',\n", - " '20190906AC',\n", - " '20190909AA',\n", - " '20190909AB',\n", - " '20190909AC',\n", - " '20190911AA',\n", - " '20190912AA',\n", - " '20190912AB',\n", - " '20190912AC',\n", - " '20190912AD',\n", - " '20190912AE',\n", - " '20190912AF',\n", - " '20190913AA',\n", - " '20190914AA',\n", - " '20190914AB',\n", - " '20190915AA',\n", - " '20190916AA',\n", - " '20190916AB',\n", - " '20190916AC',\n", - " '20190917AA',\n", - " '20190917AB',\n", - " '20190917AC',\n", - " '20190917AD',\n", - " '20190919AA',\n", - " '20190920AA',\n", - " '20190920AB',\n", - " '20190922AA',\n", - " '20190922AB',\n", - " '20190923AA',\n", - " '20190924AA',\n", - " '20190924AB',\n", - " '20190924AC',\n", - " '20190924AD',\n", - " '20190925AA',\n", - " '20190925AB',\n", - " '20190925AC',\n", - " '20190926AA',\n", - " '20190926AB',\n", - " '20190926AC',\n", - " '20190927AA',\n", - " '20190930AA',\n", - " '20190930AB',\n", - " '20190930AC',\n", - " '20191001AA',\n", - " '20191003AA',\n", - " '20191003AB',\n", - " '20191003AC',\n", - " '20191004AA',\n", - " '20191007AA',\n", - " '20191008AA',\n", - " '20191008AB',\n", - " '20191009AA',\n", - " '20191009AB',\n", - " '20191010AA',\n", - " '20191011AA',\n", - " '20191012AA',\n", - " '20191013AA',\n", - " '20191014AA',\n", - " '20191017AA',\n", - " '20191017AB',\n", - " '20191017AC',\n", - " '20191020AA',\n", - " '20191021AA',\n", - " '20191021AB',\n", - " '20191022AA',\n", - " '20191023AA',\n", - " '20191023AB',\n", - " '20191023AC',\n", - " '20191023AD',\n", - " '20191024AA',\n", - " '20191024AB',\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "create_file_tree_and_json(author_source, registry_source, metadata_directory)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "audit_files(raw_files_directory)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "convert_files(raw_files_directory, metadata_directory, directories_to_process)\n", - "print('Done converting files.')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/scripts/EGAP/files_to_import_structure.py b/scripts/EGAP/files_to_import_structure.py deleted file mode 100644 index 43b9dbfa3f16..000000000000 --- a/scripts/EGAP/files_to_import_structure.py +++ /dev/null @@ -1,159 +0,0 @@ -import os -import re -import shutil -import argparse -from distutils.dir_util import copy_tree -import logging - -from nose.tools import assert_equal - -logger = logging.getLogger(__name__) - - -# This takes the item id from the path of the project directory for example '20121001AA Findley' -> '20121001AA' -get_item_id = lambda _path: _path.split(os.sep)[-1].split(' ')[0] - - -def get_project_id(root, source_dir): - project_id_base = root.split(source_dir)[-1] - if ' ' in project_id_base: - project_id = project_id_base.split(' ')[0].split('/')[-1] - else: - project_id = project_id_base.split('/')[0] - return project_id - - -# Check if file name starts with EGAP id for example '20121001AA_PAP.pdf' -def check_id(project_id, item): - return item.startswith(project_id) - - -# Check if file follows anonymous naming convention -check_anon = lambda item: 'pap_anon' in item.lower() or 'anonymous' in item.lower() - - -def action_files_by_name(root, source_item, item_name): - """ - Pick out anonymous and create new folder to move them into it, remove ones that don't follow id naming convention. - :param root: - :param source_item: - :param item_name: - :return: - """ - project_id = get_project_id(root, source_item) - path = os.path.join(root, item_name) - if not check_id(project_id, item_name): - os.remove(path) - return - - if check_anon(item_name): - destination_parent = os.path.join('/'.join(root.split('/')[:-1]), 'anonymous') - - if not os.path.exists(destination_parent): - os.mkdir(destination_parent) - destination_item = os.path.join(destination_parent, item_name) - shutil.move(path, destination_item) - - -def audit_files(source_directory): - logger.info('Running audit. Source: {}'.format(source_directory)) - - including = open('including.txt', 'w+') - ignoring = open('ignoring.txt', 'w+') - for root, directory, files in os.walk(source_directory): - for item in files: - project_id = get_project_id(root, source_directory) - name = '{}/{}'.format(root.split(source_directory)[-1], item) # get file/folder name from just under source - if not check_id(project_id, item): - ignoring.writelines(name + '\r') - else: - including.writelines(name + '\r') - - ignoring.close() - including.close() - - projects = set(os.listdir(source_directory)) - project_ids = set([get_item_id(folders) for folders in list(projects)]) - - # check for duplicate ids - assert_equal(len(projects), len(project_ids)) - - -def main(files_dir, metadata_dir, id_list=None): - """ - This is a script for our EGAP partnership that converts the EGAP provided dump of files into a directory structure - we can easily import into the OSF. Some files in the dump are anonymous and need to be sorted into a special folder - some don't follow an id naming convention and should be ignored and not imported. - - This script copies whole file tree for a project to preserve file hierarchy then picks out anonymous files and moves - them to the anonymous folder and delete those that don't follow the naming convention. - - This script can be safely removed once all EGAP registrations have been imported. - - :param files_dir: the source path we're picking files out of - :param metadata_dir: a pre-made directory structure for importing projects that we are packing files into. - :param id_list: an optional list of project ids to limit what gets processed - :return: - """ - logger.info('Processing files. Source: {} Destination: {}'.format(files_dir, metadata_dir)) - - project_dirs = os.listdir(files_dir) - if id_list: - project_dirs = [project for project in project_dirs if get_item_id(project) in id_list] - - logger.info('Processing directories: {}'.format(project_dirs)) - - # Copy whole tree to preserve file hierarchy then - for item in project_dirs: - item_id = get_item_id(item) - source_item = os.path.join(files_dir, item) - destination_item = os.path.join(metadata_dir, item_id, 'data', 'nonanonymous') - if os.path.isdir(source_item): - copy_tree(source_item, destination_item) - - for root, directory, files in os.walk(metadata_dir): - for item in files: - if item not in ('project.json', 'registration-schema.json'): - action_files_by_name(root, metadata_dir, item) - - # Check All anon files in /anonymous/ directory - for root, directory, files in os.walk(metadata_dir): - for item in files: - if item not in ('project.json', 'registration-schema.json', '.DS_Store'): - if check_anon(item): - assert '/anonymous' in root - else: - assert '/nonanonymous' in root - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '-source', - '--source', - help='The directory for the EGAP data files, traditionally called "EGAP_REGISTRY_staging/3 Registrations/"' - ) - parser.add_argument( - '-destination', - '--destination', - help='The directory of the import file structure containing the bags of data.' - ) - parser.add_argument( - '-list', - '--list', - help='An optional list of ids to import into a the new metadata directory.' - ) - parser.add_argument( - '-audit', - '--audit', - help='Boolean to generate two lists of all files that should and should not be included. Needs "source".' - ) - - args = parser.parse_args() - source = args.source - destination = args.destination - audit = args.audit - if audit: - audit_files(source) - else: - main(source, destination) diff --git a/scripts/EGAP/requirements.txt b/scripts/EGAP/requirements.txt deleted file mode 100644 index 7e65d67ac4fc..000000000000 --- a/scripts/EGAP/requirements.txt +++ /dev/null @@ -1,68 +0,0 @@ -appnope==0.1.0 -attrs==19.3.0 -backcall==0.1.0 -bcrypt==3.1.7 -bleach==3.1.0 -blinker==1.4 -bson==0.5.8 -cffi==1.13.1 -Click==7.0 -decorator==4.4.0 -defusedxml==0.6.0 -Django==2.2.6 -django-rest-framework==0.1.0 -djangorestframework==3.10.3 -entrypoints==0.3 -Flask==1.1.1 -furl==2.1.0 -future==0.18.1 -importlib-metadata==0.23 -ipykernel==5.1.3 -ipython==7.8.0 -ipython-genutils==0.2.0 -ipywidgets==7.5.1 -itsdangerous==1.1.0 -jedi==0.15.1 -Jinja2==2.10.3 -json5==0.8.5 -jsonschema==3.1.1 -jupyter==1.0.0 -jupyter-client==5.3.4 -jupyter-console==6.0.0 -jupyter-core==4.6.1 -jupyterlab==1.1.4 -jupyterlab-server==1.0.6 -MarkupSafe==1.1.1 -mistune==0.8.4 -more-itertools==7.2.0 -nbconvert==5.6.1 -nbformat==4.4.0 -nose==1.3.7 -notebook==6.0.1 -orderedmultidict==1.0.1 -pandocfilters==1.4.2 -parso==0.5.1 -pexpect==4.7.0 -pickleshare==0.7.5 -prometheus-client==0.7.1 -prompt-toolkit==2.0.10 -ptyprocess==0.6.0 -pycparser==2.19 -Pygments==2.4.2 -pyrsistent==0.15.4 -python-dateutil==2.8.0 -pytz==2019.3 -pyzmq==18.1.0 -qtconsole==4.5.5 -Send2Trash==1.5.0 -six==1.12.0 -sqlparse==0.3.0 -terminado==0.8.2 -testpath==0.4.2 -tornado==6.0.3 -traitlets==4.3.3 -wcwidth==0.1.7 -webencodings==0.5.1 -Werkzeug==0.16.0 -widgetsnbextension==3.5.1 -zipp==0.6.0