#! /usr/bin/python

# -*- coding: utf-8 -*-

import base64

import gzip

import json

import math

import os

import pickle

import re

import shutil

# import ast

import sys

import tarfile

import time

import zipfile

import cloudpickle

import h5py

import numpy as np

import scipy.io as sio

from six.moves import cPickle

import progressbar

import tensorflow as tf

import tensorlayer as tl

from tensorflow.python.keras.saving import model_config as model_config_lib

from tensorflow.python.platform import gfile

from tensorflow.python.util import serialization

from tensorflow.python.util.tf_export import keras_export

from tensorlayer import logging, nlp, utils, visualize

import cloudpickle

import base64

from tensorflow.python.keras.saving import model_config as model_config_lib

from tensorflow.python.util.tf_export import keras_export

from tensorflow.python.util import serialization

import json

import datetime

# from six.moves import zip

if sys.version_info[0] == 2:

from urllib import urlretrieve

else:

from urllib.request import urlretrieve

# import tensorflow.contrib.eager.python.saver as tfes

# TODO: tf2.0 not stable, cannot import tensorflow.contrib.eager.python.saver

__all__ = [

'assign_weights',

'del_file',

'del_folder',

'download_file_from_google_drive',

'exists_or_mkdir',

'file_exists',

'folder_exists',

'load_and_assign_npz',

'load_and_assign_npz_dict',

'load_ckpt',

'load_cropped_svhn',

'load_file_list',

'load_folder_list',

'load_npy_to_any',

'load_npz',

'maybe_download_and_extract',

'natural_keys',

'npz_to_W_pdf',

'read_file',

'save_any_to_npy',

'save_ckpt',

'save_npz',

'save_npz_dict',

'tf_variables_to_numpy',

'assign_tf_variable',

'save_weights_to_hdf5',

'load_hdf5_to_weights_in_order',

'load_hdf5_to_weights',

'save_hdf5_graph',

'load_hdf5_graph',

# 'net2static_graph',

'static_graph2net',

# 'save_pkl_graph',

# 'load_pkl_graph',

]

def func2str(expr):

b = cloudpickle.dumps(expr)

s = base64.b64encode(b).decode()

return s

def str2func(s):

b = base64.b64decode(s)

expr = cloudpickle.loads(b)

return expr

# def net2static_graph(network):

# saved_file = dict()

# # if network._NameNone is True:

# # saved_file.update({"name": None})

# # else:

# # saved_file.update({"name": network.name})

# # if not isinstance(network.inputs, list):

# # saved_file.update({"inputs": network.inputs._info[0].name})

# # else:

# # saved_inputs = []

# # for saved_input in network.inputs:

# # saved_inputs.append(saved_input._info[0].name)

# # saved_file.update({"inputs": saved_inputs})

# # if not isinstance(network.outputs, list):

# # saved_file.update({"outputs": network.outputs._info[0].name})

# # else:

# # saved_outputs = []

# # for saved_output in network.outputs:

# # saved_outputs.append(saved_output._info[0].name)

# # saved_file.update({"outputs": saved_outputs})

# saved_file.update({"config": network.config})

#

# return saved_file

@keras_export('keras.models.save_model')

def save_keras_model(model):

# f.attrs['keras_model_config'] = json.dumps(

# {

# 'class_name': model.__class__.__name__,

# 'config': model.get_config()

# },

# default=serialization.get_json_type).encode('utf8')

#

# f.flush()

return json.dumps(

{

'class_name': model.__class__.__name__,

'config': model.get_config()

}, default=serialization.get_json_type

).encode('utf8')

@keras_export('keras.models.load_model')

def load_keras_model(model_config):

custom_objects = {}

if model_config is None:

raise ValueError('No model found in config.')

model_config = json.loads(model_config.decode('utf-8'))

model = model_config_lib.model_from_config(model_config, custom_objects=custom_objects)

return model

def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False, customized_data=None):

"""Save the architecture of TL model into a hdf5 file. Support saving model weights.

Parameters

-----------

network : TensorLayer Model.

The network to save.

filepath : str

The name of model file.

save_weights : bool

Whether to save model weights.

customized_data : dict

The user customized meta data.

Examples

--------

>>> # Save the architecture (with parameters)

>>> tl.files.save_hdf5_graph(network, filepath='model.hdf5', save_weights=True)

>>> # Save the architecture (without parameters)

>>> tl.files.save_hdf5_graph(network, filepath='model.hdf5', save_weights=False)

>>> # Load the architecture in another script (no parameters restore)

>>> net = tl.files.load_hdf5_graph(filepath='model.hdf5', load_weights=False)

>>> # Load the architecture in another script (restore parameters)

>>> net = tl.files.load_hdf5_graph(filepath='model.hdf5', load_weights=True)

"""

if network.outputs is None:

raise RuntimeError("save_hdf5_graph not support dynamic mode yet")

logging.info("[*] Saving TL model into {}, saving weights={}".format(filepath, save_weights))

model_config = network.config # net2static_graph(network)

model_config["version_info"]["save_date"] = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc

).isoformat()

model_config_str = str(model_config)

customized_data_str = str(customized_data)

# version_info = {

# "tensorlayer_version": tl.__version__,

# "backend": "tensorflow",

# "backend_version": tf.__version__,

# "training_device": "gpu",

# "save_date": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()

# }

# version_info_str = str(version_info)

with h5py.File(filepath, 'w') as f:

f.attrs["model_config"] = model_config_str.encode('utf8')

f.attrs["customized_data"] = customized_data_str.encode('utf8')

# f.attrs["version_info"] = version_info_str.encode('utf8')

if save_weights:

_save_weights_to_hdf5_group(f, network.all_layers)

f.flush()

logging.info("[*] Saved TL model into {}, saving weights={}".format(filepath, save_weights))

def generate_func(args):

for key in args:

if isinstance(args[key], tuple) and args[key][0] == 'is_Func':

fn = str2func(args[key][1])

args[key] = fn

# if key in ['act']:

# # fn_dict = args[key]

# # module_path = fn_dict['module_path']

# # func_name = fn_dict['func_name']

# # lib = importlib.import_module(module_path)

# # fn = getattr(lib, func_name)

# # args[key] = fn

# fn = str2func(args[key])

# args[key] = fn

# elif key in ['fn']:

# fn = str2func(args[key])

# args[key] = fn

def eval_layer(layer_kwargs):

layer_class = layer_kwargs.pop('class')

args = layer_kwargs['args']

layer_type = args.pop('layer_type')

if layer_type == "normal":

generate_func(args)

return eval('tl.layers.' + layer_class)(**args)

elif layer_type == "layerlist":

ret_layer = []

layers = args["layers"]

for layer_graph in layers:

ret_layer.append(eval_layer(layer_graph))

args['layers'] = ret_layer

return eval('tl.layers.' + layer_class)(**args)

elif layer_type == "modellayer":

M = static_graph2net(args['model'])

args['model'] = M

return eval('tl.layers.' + layer_class)(**args)

elif layer_type == "keraslayer":

M = load_keras_model(args['fn'])

input_shape = args.pop('keras_input_shape')

_ = M(np.random.random(input_shape).astype(np.float32))

args['fn'] = M

args['fn_weights'] = M.trainable_variables

return eval('tl.layers.' + layer_class)(**args)

else:

raise RuntimeError("Unknown layer type.")

def static_graph2net(model_config):

layer_dict = {}

model_name = model_config["name"]

inputs_tensors = model_config["inputs"]

outputs_tensors = model_config["outputs"]

all_args = model_config["model_architecture"]

for idx, layer_kwargs in enumerate(all_args):

layer_class = layer_kwargs["class"] # class of current layer

prev_layers = layer_kwargs.pop("prev_layer") # name of previous layers

net = eval_layer(layer_kwargs)

if layer_class in tl.layers.inputs.__all__:

net = net._nodes[0].out_tensors[0]

if prev_layers is not None:

for prev_layer in prev_layers:

if not isinstance(prev_layer, list):

output = net(layer_dict[prev_layer])

layer_dict[output._info[0].name] = output

else:

list_layers = [layer_dict[layer] for layer in prev_layer]

output = net(list_layers)

layer_dict[output._info[0].name] = output

else:

layer_dict[net._info[0].name] = net

if not isinstance(inputs_tensors, list):

model_inputs = layer_dict[inputs_tensors]

else:

model_inputs = []

for inputs_tensor in inputs_tensors:

model_inputs.append(layer_dict[inputs_tensor])

if not isinstance(outputs_tensors, list):

model_outputs = layer_dict[outputs_tensors]

else:

model_outputs = []

for outputs_tensor in outputs_tensors:

model_outputs.append(layer_dict[outputs_tensor])

from tensorlayer.models import Model

M = Model(inputs=model_inputs, outputs=model_outputs, name=model_name)

logging.info("[*] Load graph finished")

return M

def load_hdf5_graph(filepath='model.hdf5', load_weights=False):

"""Restore TL model archtecture from a a pickle file. Support loading model weights.

Parameters

-----------

filepath : str

The name of model file.

load_weights : bool

Whether to load model weights.

Returns

--------

network : TensorLayer Model.

Examples

--------

- see ``tl.files.save_hdf5_graph``

"""

logging.info("[*] Loading TL model from {}, loading weights={}".format(filepath, load_weights))

f = h5py.File(filepath, 'r')

model_config_str = f.attrs["model_config"].decode('utf8')

model_config = eval(model_config_str)

# version_info_str = f.attrs["version_info"].decode('utf8')

# version_info = eval(version_info_str)

version_info = model_config["version_info"]

backend_version = version_info["backend_version"]

tensorlayer_version = version_info["tensorlayer_version"]

if backend_version != tf.__version__:

logging.warning(

"Saved model uses tensorflow version {}, but now you are using tensorflow version {}".format(

backend_version, tf.__version__

)

)

if tensorlayer_version != tl.__version__:

logging.warning(

"Saved model uses tensorlayer version {}, but now you are using tensorlayer version {}".format(

tensorlayer_version, tl.__version__

)

)

M = static_graph2net(model_config)

if load_weights:

if not ('layer_names' in f.attrs.keys()):

raise RuntimeError("Saved model does not contain weights.")

M.load_weights(filepath=filepath)

f.close()

logging.info("[*] Loaded TL model from {}, loading weights={}".format(filepath, load_weights))

return M

# def load_pkl_graph(name='model.pkl'):

# """Restore TL model archtecture from a a pickle file. No parameters be restored.

#

# Parameters

# -----------

# name : str

# The name of graph file.

#

# Returns

# --------

# network : TensorLayer Model.

#

# Examples

# --------

# >>> # It is better to use load_hdf5_graph

# """

# logging.info("[*] Loading TL graph from {}".format(name))

# with open(name, 'rb') as file:

# saved_file = pickle.load(file)

#

# M = static_graph2net(saved_file)

#

# return M

#

#

# def save_pkl_graph(network, name='model.pkl'):

# """Save the architecture of TL model into a pickle file. No parameters be saved.

#

# Parameters

# -----------

# network : TensorLayer layer

# The network to save.

# name : str

# The name of graph file.

#

# Example

# --------

# >>> # It is better to use save_hdf5_graph

# """

# if network.outputs is None:

# raise AssertionError("save_graph not support dynamic mode yet")

#

# logging.info("[*] Saving TL graph into {}".format(name))

#

# saved_file = net2static_graph(network)

#

# with open(name, 'wb') as file:

# pickle.dump(saved_file, file, protocol=pickle.HIGHEST_PROTOCOL)

# logging.info("[*] Saved graph")

# Load dataset functions

def load_mnist_dataset(shape=(-1, 784), path='data'):

"""Load the original mnist.

Automatically download MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 digit images respectively.

Parameters

----------

shape : tuple

The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).

path : str

The path that the data is downloaded to.

Returns

-------

X_train, y_train, X_val, y_val, X_test, y_test: tuple

Return splitted training/validation/test set respectively.

Examples

--------

>>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1,784), path='datasets')

>>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 28, 28, 1))

"""

return _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/')

def load_fashion_mnist_dataset(shape=(-1, 784), path='data'):

"""Load the fashion mnist.

Automatically download fashion-MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 fashion images respectively, `examples <http://marubon-ds.blogspot.co.uk/2017/09/fashion-mnist-exploring.html>`__.

Parameters

----------

shape : tuple

The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).

path : str

The path that the data is downloaded to.

Returns

-------

X_train, y_train, X_val, y_val, X_test, y_test: tuple

Return splitted training/validation/test set respectively.

Examples

--------

>>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1,784), path='datasets')

>>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1, 28, 28, 1))

"""

return _load_mnist_dataset(

shape, path, name='fashion_mnist', url='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/'

)

def _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):

"""A generic function to load mnist-like dataset.

Parameters:

----------

shape : tuple

The shape of digit images.

path : str

The path that the data is downloaded to.

name : str

The dataset name you want to use(the default is 'mnist').

url : str

The url of dataset(the default is 'http://yann.lecun.com/exdb/mnist/').

"""

path = os.path.join(path, name)

# Define functions for loading mnist-like data's images and labels.

# For convenience, they also download the requested files if needed.

def load_mnist_images(path, filename):

filepath = maybe_download_and_extract(filename, path, url)

logging.info(filepath)

# Read the inputs in Yann LeCun's binary format.

with gzip.open(filepath, 'rb') as f:

data = np.frombuffer(f.read(), np.uint8, offset=16)

# The inputs are vectors now, we reshape them to monochrome 2D images,

# following the shape convention: (examples, channels, rows, columns)

data = data.reshape(shape)

# The inputs come as bytes, we convert them to float32 in range [0,1].

# (Actually to range [0, 255/256], for compatibility to the version

# provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)

return data / np.float32(256)

def load_mnist_labels(path, filename):

filepath = maybe_download_and_extract(filename, path, url)

# Read the labels in Yann LeCun's binary format.

with gzip.open(filepath, 'rb') as f:

data = np.frombuffer(f.read(), np.uint8, offset=8)

# The labels are vectors of integers now, that's exactly what we want.

return data

# Download and read the training and test set images and labels.

logging.info("Load or Download {0} > {1}".format(name.upper(), path))

X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')

y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')

X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')

y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')

# We reserve the last 10000 training examples for validation.

X_train, X_val = X_train[:-10000], X_train[-10000:]

y_train, y_val = y_train[:-10000], y_train[-10000:]

# We just return all the arrays in order, as expected in main().

# (It doesn't matter how we do this as long as we can read them again.)

X_train = np.asarray(X_train, dtype=np.float32)

y_train = np.asarray(y_train, dtype=np.int32)

X_val = np.asarray(X_val, dtype=np.float32)

y_val = np.asarray(y_val, dtype=np.int32)

X_test = np.asarray(X_test, dtype=np.float32)

y_test = np.asarray(y_test, dtype=np.int32)

return X_train, y_train, X_val, y_val, X_test, y_test

def load_cifar10_dataset(shape=(-1, 32, 32, 3), path='data', plotable=False):

"""Load CIFAR-10 dataset.

It consists of 60000 32x32 colour images in 10 classes, with

6000 images per class. There are 50000 training images and 10000 test images.

The dataset is divided into five training batches and one test batch, each with

10000 images. The test batch contains exactly 1000 randomly-selected images from

each class. The training batches contain the remaining images in random order,

but some training batches may contain more images from one class than another.

Between them, the training batches contain exactly 5000 images from each class.

Parameters

----------

shape : tupe

The shape of digit images e.g. (-1, 3, 32, 32) and (-1, 32, 32, 3).

path : str

The path that the data is downloaded to, defaults is ``data/cifar10/``.

plotable : boolean

Whether to plot some image examples, False as default.

Examples

--------

>>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))

References

----------

- `CIFAR website <https://www.cs.toronto.edu/~kriz/cifar.html>`__

- `Data download link <https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz>`__

- `<https://teratail.com/questions/28932>`__

"""

path = os.path.join(path, 'cifar10')

logging.info("Load or Download cifar10 > {}".format(path))

# Helper function to unpickle the data

def unpickle(file):

fp = open(file, 'rb')

if sys.version_info.major == 2:

data = pickle.load(fp)

elif sys.version_info.major == 3:

data = pickle.load(fp, encoding='latin-1')

fp.close()

return data

filename = 'cifar-10-python.tar.gz'

url = 'https://www.cs.toronto.edu/~kriz/'

# Download and uncompress file

maybe_download_and_extract(filename, path, url, extract=True)

# Unpickle file and fill in data

X_train = None

y_train = []

for i in range(1, 6):

data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "data_batch_{}".format(i)))

if i == 1:

X_train = data_dic['data']

else:

X_train = np.vstack((X_train, data_dic['data']))

y_train += data_dic['labels']

test_data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "test_batch"))

X_test = test_data_dic['data']

y_test = np.array(test_data_dic['labels'])

if shape == (-1, 3, 32, 32):

X_test = X_test.reshape(shape)

X_train = X_train.reshape(shape)

elif shape == (-1, 32, 32, 3):

X_test = X_test.reshape(shape, order='F')

X_train = X_train.reshape(shape, order='F')

X_test = np.transpose(X_test, (0, 2, 1, 3))

X_train = np.transpose(X_train, (0, 2, 1, 3))

else:

X_test = X_test.reshape(shape)

X_train = X_train.reshape(shape)

y_train = np.array(y_train)

if plotable:

if sys.platform.startswith('darwin'):

import matplotlib

matplotlib.use('TkAgg')

import matplotlib.pyplot as plt

logging.info('\nCIFAR-10')

fig = plt.figure(1)

logging.info('Shape of a training image: X_train[0] %s' % X_train[0].shape)

plt.ion() # interactive mode

count = 1

for _ in range(10): # each row

for _ in range(10): # each column

_ = fig.add_subplot(10, 10, count)

if shape == (-1, 3, 32, 32):

# plt.imshow(X_train[count-1], interpolation='nearest')

plt.imshow(np.transpose(X_train[count - 1], (1, 2, 0)), interpolation='nearest')

# plt.imshow(np.transpose(X_train[count-1], (2, 1, 0)), interpolation='nearest')

elif shape == (-1, 32, 32, 3):

plt.imshow(X_train[count - 1], interpolation='nearest')

# plt.imshow(np.transpose(X_train[count-1], (1, 0, 2)), interpolation='nearest')

else:

raise Exception("Do not support the given 'shape' to plot the image examples")

plt.gca().xaxis.set_major_locator(plt.NullLocator()) # 不显示刻度(tick)

plt.gca().yaxis.set_major_locator(plt.NullLocator())

count = count + 1

plt.draw() # interactive mode

plt.pause(3) # interactive mode

logging.info("X_train: %s" % X_train.shape)

logging.info("y_train: %s" % y_train.shape)

logging.info("X_test: %s" % X_test.shape)

logging.info("y_test: %s" % y_test.shape)

X_train = np.asarray(X_train, dtype=np.float32)

X_test = np.asarray(X_test, dtype=np.float32)

y_train = np.asarray(y_train, dtype=np.int32)

y_test = np.asarray(y_test, dtype=np.int32)

return X_train, y_train, X_test, y_test

def load_cropped_svhn(path='data', include_extra=True):

"""Load Cropped SVHN.

The Cropped Street View House Numbers (SVHN) Dataset contains 32x32x3 RGB images.

Digit '1' has label 1, '9' has label 9 and '0' has label 0 (the original dataset uses 10 to represent '0'), see `ufldl website <http://ufldl.stanford.edu/housenumbers/>`__.

Parameters

----------

path : str

The path that the data is downloaded to.

include_extra : boolean

If True (default), add extra images to the training set.

Returns

-------

X_train, y_train, X_test, y_test: tuple

Return splitted training/test set respectively.

Examples

---------

>>> X_train, y_train, X_test, y_test = tl.files.load_cropped_svhn(include_extra=False)

>>> tl.vis.save_images(X_train[0:100], [10, 10], 'svhn.png')

"""

start_time = time.time()

path = os.path.join(path, 'cropped_svhn')

logging.info("Load or Download Cropped SVHN > {} | include extra images: {}".format(path, include_extra))

url = "http://ufldl.stanford.edu/housenumbers/"

np_file = os.path.join(path, "train_32x32.npz")

if file_exists(np_file) is False:

filename = "train_32x32.mat"

filepath = maybe_download_and_extract(filename, path, url)

mat = sio.loadmat(filepath)

X_train = mat['X'] / 255.0 # to [0, 1]

X_train = np.transpose(X_train, (3, 0, 1, 2))

y_train = np.squeeze(mat['y'], axis=1)

y_train[y_train == 10] = 0 # replace 10 to 0

np.savez(np_file, X=X_train, y=y_train)

del_file(filepath)

else:

v = np.load(np_file)

X_train = v['X']

y_train = v['y']

logging.info(" n_train: {}".format(len(y_train)))

np_file = os.path.join(path, "test_32x32.npz")

if file_exists(np_file) is False:

filename = "test_32x32.mat"

filepath = maybe_download_and_extract(filename, path, url)

mat = sio.loadmat(filepath)

X_test = mat['X'] / 255.0

X_test = np.transpose(X_test, (3, 0, 1, 2))

y_test = np.squeeze(mat['y'], axis=1)

y_test[y_test == 10] = 0

np.savez(np_file, X=X_test, y=y_test)

del_file(filepath)

else:

v = np.load(np_file)

X_test = v['X']

y_test = v['y']

logging.info(" n_test: {}".format(len(y_test)))

if include_extra:

logging.info(" getting extra 531131 images, please wait ...")

np_file = os.path.join(path, "extra_32x32.npz")

if file_exists(np_file) is False:

logging.info(" the first time to load extra images will take long time to convert the file format ...")

filename = "extra_32x32.mat"

filepath = maybe_download_and_extract(filename, path, url)

mat = sio.loadmat(filepath)

X_extra = mat['X'] / 255.0

X_extra = np.transpose(X_extra, (3, 0, 1, 2))

y_extra = np.squeeze(mat['y'], axis=1)

y_extra[y_extra == 10] = 0

np.savez(np_file, X=X_extra, y=y_extra)

del_file(filepath)

else:

v = np.load(np_file)

X_extra = v['X']

y_extra = v['y']

# print(X_train.shape, X_extra.shape)

logging.info(" adding n_extra {} to n_train {}".format(len(y_extra), len(y_train)))

t = time.time()

X_train = np.concatenate((X_train, X_extra), 0)

y_train = np.concatenate((y_train, y_extra), 0)

# X_train = np.append(X_train, X_extra, axis=0)

# y_train = np.append(y_train, y_extra, axis=0)

logging.info(" added n_extra {} to n_train {} took {}s".format(len(y_extra), len(y_train), time.time() - t))

else:

logging.info(" no extra images are included")

logging.info(" image size: %s n_train: %d n_test: %d" % (str(X_train.shape[1:4]), len(y_train), len(y_test)))

logging.info(" took: {}s".format(int(time.time() - start_time)))

return X_train, y_train, X_test, y_test

def load_ptb_dataset(path='data'):

"""Load Penn TreeBank (PTB) dataset.

It is used in many LANGUAGE MODELING papers,

including "Empirical Evaluation and Combination of Advanced Language

Modeling Techniques", "Recurrent Neural Network Regularization".

It consists of 929k training words, 73k validation words, and 82k test

words. It has 10k words in its vocabulary.

Parameters

----------

path : str

The path that the data is downloaded to, defaults is ``data/ptb/``.

Returns

--------

train_data, valid_data, test_data : list of int

The training, validating and testing data in integer format.

vocab_size : int

The vocabulary size.

Examples

--------

>>> train_data, valid_data, test_data, vocab_size = tl.files.load_ptb_dataset()

References

---------------

- ``tensorflow.models.rnn.ptb import reader``

- `Manual download <http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz>`__

Notes

------

- If you want to get the raw data, see the source code.

"""

path = os.path.join(path, 'ptb')

logging.info("Load or Download Penn TreeBank (PTB) dataset > {}".format(path))

# Maybe dowload and uncompress tar, or load exsisting files

filename = 'simple-examples.tgz'

url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'

maybe_download_and_extract(filename, path, url, extract=True)

data_path = os.path.join(path, 'simple-examples', 'data')

train_path = os.path.join(data_path, "ptb.train.txt")

valid_path = os.path.join(data_path, "ptb.valid.txt")

test_path = os.path.join(data_path, "ptb.test.txt")

word_to_id = nlp.build_vocab(nlp.read_words(train_path))

train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)

valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)

test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)

vocab_size = len(word_to_id)

# logging.info(nlp.read_words(train_path)) # ... 'according', 'to', 'mr.', '<unk>', '<eos>']

# logging.info(train_data) # ... 214, 5, 23, 1, 2]

# logging.info(word_to_id) # ... 'beyond': 1295, 'anti-nuclear': 9599, 'trouble': 1520, '<eos>': 2 ... }

# logging.info(vocabulary) # 10000

# exit()

return train_data, valid_data, test_data, vocab_size

def load_matt_mahoney_text8_dataset(path='data'):

"""Load Matt Mahoney's dataset.

Download a text file from Matt Mahoney's website

if not present, and make sure it's the right size.

Extract the first file enclosed in a zip file as a list of words.

This dataset can be used for Word Embedding.

Parameters

----------

path : str

The path that the data is downloaded to, defaults is ``data/mm_test8/``.

Returns

--------

list of str

The raw text data e.g. [.... 'their', 'families', 'who', 'were', 'expelled', 'from', 'jerusalem', ...]

Examples

--------

>>> words = tl.files.load_matt_mahoney_text8_dataset()

>>> print('Data size', len(words))

"""

path = os.path.join(path, 'mm_test8')

logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))

filename = 'text8.zip'

url = 'http://mattmahoney.net/dc/'

maybe_download_and_extract(filename, path, url, expected_bytes=31344016)

with zipfile.ZipFile(os.path.join(path, filename)) as f:

word_list = f.read(f.namelist()[0]).split()

for idx, _ in enumerate(word_list):

word_list[idx] = word_list[idx].decode()

return word_list

def load_imdb_dataset(

path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,

index_from=3

):

"""Load IMDB dataset.

Parameters

----------

path : str

The path that the data is downloaded to, defaults is ``data/imdb/``.

nb_words : int

Number of words to get.

skip_top : int

Top most frequent words to ignore (they will appear as oov_char value in the sequence data).

maxlen : int

Maximum sequence length. Any longer sequence will be truncated.

seed : int

Seed for reproducible data shuffling.

start_char : int

The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.

oov_char : int

Words that were cut out because of the num_words or skip_top limit will be replaced with this character.

index_from : int

Index actual words with this index and higher.

Examples

--------

>>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(

... nb_words=20000, test_split=0.2)

>>> print('X_train.shape', X_train.shape)

(20000,) [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]

>>> print('y_train.shape', y_train.shape)

(20000,) [1 0 0 ..., 1 0 1]

References

-----------

- `Modified from keras. <https://github.com/fchollet/keras/blob/master/keras/datasets/imdb.py>`__

"""

path = os.path.join(path, 'imdb')

filename = "imdb.pkl"

url = 'https://s3.amazonaws.com/text-datasets/'

maybe_download_and_extract(filename, path, url)

if filename.endswith(".gz"):

f = gzip.open(os.path.join(path, filename), 'rb')

else:

f = open(os.path.join(path, filename), 'rb')

X, labels = cPickle.load(f)

f.close()

np.random.seed(seed)

np.random.shuffle(X)

np.random.seed(seed)

np.random.shuffle(labels)

if start_char is not None:

X = [[start_char] + [w + index_from for w in x] for x in X]

elif index_from:

X = [[w + index_from for w in x] for x in X]

if maxlen:

new_X = []

new_labels = []

for x, y in zip(X, labels):

if len(x) < maxlen:

new_X.append(x)

new_labels.append(y)

X = new_X

labels = new_labels

if not X:

raise Exception(

'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '

'Increase maxlen.'

)

if not nb_words:

nb_words = max([max(x) for x in X])

# by convention, use 2 as OOV word

# reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)

if oov_char is not None:

X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]

else:

nX = []

for x in X:

nx = []

for w in x:

if (w >= nb_words or w < skip_top):

nx.append(w)

nX.append(nx)

X = nX

X_train = np.array(X[:int(len(X) * (1 - test_split))])

y_train = np.array(labels[:int(len(X) * (1 - test_split))])

X_test = np.array(X[int(len(X) * (1 - test_split)):])

y_test = np.array(labels[int(len(X) * (1 - test_split)):])

return X_train, y_train, X_test, y_test

def load_nietzsche_dataset(path='data'):

"""Load Nietzsche dataset.

Parameters

----------

path : str

The path that the data is downloaded to, defaults is ``data/nietzsche/``.

Returns

--------

str

The content.

Examples

--------

>>> see tutorial_generate_text.py

>>> words = tl.files.load_nietzsche_dataset()

>>> words = basic_clean_str(words)

>>> words = words.split()

"""

logging.info("Load or Download nietzsche dataset > {}".format(path))

path = os.path.join(path, 'nietzsche')

filename = "nietzsche.txt"

url = 'https://s3.amazonaws.com/text-datasets/'

filepath = maybe_download_and_extract(filename, path, url)

with open(filepath, "r") as f:

words = f.read()

return words

def load_wmt_en_fr_dataset(path='data'):

"""Load WMT'15 English-to-French translation dataset.

It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.

Returns the directories of training data and test data.

Parameters

----------

path : str

The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.

References

----------

- Code modified from /tensorflow/models/rnn/translation/data_utils.py

Notes

-----