add setup file for pip installable

requirements.txt

@@ -0,0 +1 @@
+tensorflow>=1.0.0

setup.py

@@ -0,0 +1,25 @@
+import setuptools
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="xlnet",
+    version="0.0.1",
+    author="zihangdai",
+    author_email="[email protected]",
+    description="XLNet: Generalized Autoregressive Pretraining for Language Understanding",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/zihangdai/xlnet",
+    packages=setuptools.find_packages(),
+    install_requires=[
+         'tensorflow'
+    ],
+    include_package_data=True,
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache-2.0 License",
+        "Operating System :: OS Independent",
+    ],
+)

classifier_utils.py → xlnet/classifier_utils.py

@@ -4,7 +4,7 @@
 import numpy as np
 
 import tensorflow as tf
-from data_utils import SEP_ID, CLS_ID
+from xlnet.data_utils import SEP_ID, CLS_ID
 
 FLAGS = flags.FLAGS
 

data_utils.py → xlnet/data_utils.py

@@ -16,7 +16,7 @@
 
 import tensorflow as tf
 
-from prepro_utils import preprocess_text, encode_ids
+from xlnet.prepro_utils import preprocess_text, encode_ids
 import sentencepiece as spm
 
 

function_builder.py → xlnet/function_builder.py

@@ -3,10 +3,9 @@
 from __future__ import division
 from __future__ import print_function
 
-import functools
 import os
 import tensorflow as tf
-import modeling
+from xlnet import modeling
 import xlnet
 
 

run_race.py → xlnet/run_race.py

@@ -2,33 +2,21 @@
 from __future__ import division
 from __future__ import print_function
 
-from os.path import join
 from absl import flags
 import os
-import csv
 import collections
 import numpy as np
-import time
-import math
 import json
 import random
-from copy import copy
-from collections import defaultdict as dd
-
-from scipy.stats import pearsonr, spearmanr
-from sklearn.metrics import matthews_corrcoef, f1_score
-
-import absl.logging as _logging  # pylint: disable=unused-import
 
 import tensorflow as tf
 import sentencepiece as spm
 
-from data_utils import SEP_ID, VOCAB_SIZE, CLS_ID
-import model_utils
-import function_builder
-from classifier_utils import PaddingInputExample
-from classifier_utils import convert_single_example
-from prepro_utils import preprocess_text, encode_ids
+from xlnet.data_utils import SEP_ID, CLS_ID
+from xlnet import model_utils
+from xlnet import function_builder
+from xlnet.classifier_utils import PaddingInputExample, convert_single_example
+from xlnet.prepro_utils import preprocess_text, encode_ids
 
 # Model
 flags.DEFINE_string("model_config_path", default=None,

run_squad.py → xlnet/run_squad.py

@@ -8,7 +8,6 @@
 
 import collections
 import os
-import time
 import math
 import json
 import six
@@ -24,11 +23,11 @@
 
 import tensorflow as tf
 import sentencepiece as spm
-from prepro_utils import preprocess_text, encode_ids, encode_pieces, printable_text
-import function_builder
-import model_utils
-import squad_utils
-from data_utils import SEP_ID, CLS_ID, VOCAB_SIZE
+from xlnet.prepro_utils import preprocess_text, encode_ids, encode_pieces, printable_text
+from xlnet import function_builder
+from xlnet import model_utils
+from xlnet import squad_utils
+from xlnet.data_utils import SEP_ID, CLS_ID,
 
 SPIECE_UNDERLINE = u'▁'
 

train.py → xlnet/train.py

@@ -12,10 +12,10 @@
 import numpy as np
 
 import tensorflow as tf
-import model_utils
-import tpu_estimator
-import function_builder
-import data_utils
+from xlnet import model_utils
+from xlnet import tpu_estimator
+from xlnet import function_builder
+from xlnet import data_utils
 
 # TPU parameters
 flags.DEFINE_string("master", default=None,

train_gpu.py → xlnet/train_gpu.py

@@ -14,10 +14,10 @@
 
 import tensorflow as tf
 
-import data_utils
-import model_utils
-from gpu_utils import assign_to_gpu, average_grads_and_vars
-import function_builder
+from xlnet import data_utils
+from xlnet import model_utils
+from xlnet.gpu_utils import assign_to_gpu, average_grads_and_vars
+from xlnet import function_builder
 
 
 # GPU config

xlnet.py → xlnet/xlnet.py

@@ -5,7 +5,7 @@
 import json
 import os
 import tensorflow as tf
-import modeling
+from xlnet import modeling
 
 
 def _get_initializer(FLAGS):
@@ -28,7 +28,6 @@ class XLNetConfig(object):
   """XLNetConfig contains hyperparameters that are specific to a model checkpoint;
   i.e., these hyperparameters should be the same between
   pretraining and finetuning.
-
   The following hyperparameters are defined:
     n_layer: int, the number of layers.
     d_model: int, the hidden size.
@@ -232,7 +231,6 @@ def get_pooled_out(self, summary_type, use_summ_proj=True):
       summary_type: str, "last", "first", "mean", or "attn". The method
         to pool the input to get a vector representation.
       use_summ_proj: bool, whether to use a linear projection during pooling.
-
     Returns:
       float32 Tensor in shape [bsz, d_model], the pooled representation.
     """
@@ -288,5 +286,4 @@ def get_initializer(self):
     Returns:
       A tf initializer. Used to initialize variables in layers on top of XLNet.
     """
-    return self.initializer
-
+    return self.initializer