resolve dropout bugs

Author: dragen1860

inits.py

@@ -4,14 +4,14 @@
 
 def uniform(shape, scale=0.05, name=None):
     """Uniform init."""
-    initial = tf.random_uniform(shape, minval=-scale, maxval=scale, dtype=tf.float32)
+    initial = tf.random.uniform(shape, minval=-scale, maxval=scale, dtype=tf.float32)
     return tf.Variable(initial, name=name)
 
 
 def glorot(shape, name=None):
     """Glorot & Bengio (AISTATS 2010) init."""
     init_range = np.sqrt(6.0/(shape[0]+shape[1]))
-    initial = tf.random_uniform(shape, minval=-init_range, maxval=init_range, dtype=tf.float32)
+    initial = tf.random.uniform(shape, minval=-init_range, maxval=init_range, dtype=tf.float32)
     return tf.Variable(initial, name=name)
 
 

layers.py

@@ -21,15 +21,15 @@ def get_layer_uid(layer_name=''):
         return _LAYER_UIDS[layer_name]
 
 
-def sparse_dropout(x, keep_prob, noise_shape):
+def sparse_dropout(x, rate, noise_shape):
     """
     Dropout for sparse tensors.
     """
-    random_tensor = keep_prob
+    random_tensor = 1 - rate
     random_tensor += tf.random.uniform(noise_shape)
     dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool)
     pre_out = tf.sparse.retain(x, dropout_mask)
-    return pre_out * (1./keep_prob)
+    return pre_out * (1./(1 - rate))
 
 
 def dot(x, y, sparse=False):
@@ -43,54 +43,9 @@ def dot(x, y, sparse=False):
     return res
 
 
-class Layer(object):
-    """
-    Base layer class. Defines basic API for all layer objects.
-    Implementation inspired by keras (http://keras.io).
-
-    # Properties
-        name: String, defines the variable scope of the layer.
-        logging: Boolean, switches Tensorflow histogram logging on/off
-
-    # Methods
-        _call(inputs): Defines computation graph of layer
-            (i.e. takes input, returns output)
-        __call__(inputs): Wrapper for _call()
-        _log_vars(): Log all variables
-    """
-
-    def __init__(self, **kwargs):
-        allowed_kwargs = {'name', 'logging'}
-        for kwarg in kwargs.keys():
-            assert kwarg in allowed_kwargs, 'Invalid keyword argument: ' + kwarg
-        name = kwargs.get('name')
-        if not name:
-            layer = self.__class__.__name__.lower()
-            name = layer + '_' + str(get_layer_uid(layer))
-        self.name = name
-        self.vars = {}
-        logging = kwargs.get('logging', False)
-        self.logging = logging
-        self.sparse_inputs = False
-
-    def _call(self, inputs):
-        return inputs
-
-    def __call__(self, inputs):
-        with tf.name_scope(self.name):
-            if self.logging and not self.sparse_inputs:
-                tf.summary.histogram(self.name + '/inputs', inputs)
-            outputs = self._call(inputs)
-            if self.logging:
-                tf.summary.histogram(self.name + '/outputs', outputs)
-            return outputs
 
-    def _log_vars(self):
-        for var in self.vars:
-            tf.summary.histogram(self.name + '/vars/' + var, self.vars[var])
 
-
-class Dense(Layer):
+class Dense(layers.Layer):
     """Dense layer."""
     def __init__(self, input_dim, output_dim, placeholders, dropout=0., sparse_inputs=False,
                  act=tf.nn.relu, bias=False, featureless=False, **kwargs):
@@ -169,9 +124,9 @@ def call(self, inputs, training=None):
         x, support_ = inputs
 
         # dropout
-        if self.is_sparse_inputs:
+        if training and self.is_sparse_inputs:
             x = sparse_dropout(x, self.dropout, self.num_features_nonzero)
-        else:
+        elif training:
             x = tf.nn.dropout(x, self.dropout)
 
 

metrics.py

@@ -2,7 +2,9 @@
 
 
 def masked_softmax_cross_entropy(preds, labels, mask):
-    """Softmax cross-entropy loss with masking."""
+    """
+    Softmax cross-entropy loss with masking.
+    """
     loss = tf.nn.softmax_cross_entropy_with_logits(logits=preds, labels=labels)
     mask = tf.cast(mask, dtype=tf.float32)
     mask /= tf.reduce_mean(mask)
@@ -11,7 +13,9 @@ def masked_softmax_cross_entropy(preds, labels, mask):
 
 
 def masked_accuracy(preds, labels, mask):
-    """Accuracy with masking."""
+    """
+    Accuracy with masking.
+    """
     correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(labels, 1))
     accuracy_all = tf.cast(correct_prediction, tf.float32)
     mask = tf.cast(mask, dtype=tf.float32)

models.py

@@ -5,86 +5,10 @@
 from    config import args 
 
 
-class Model:
-    def __init__(self, **kwargs):
-        allowed_kwargs = {'name', 'logging'}
-        for kwarg in kwargs.keys():
-            assert kwarg in allowed_kwargs, 'Invalid keyword argument: ' + kwarg
-        name = kwargs.get('name')
-        if not name:
-            name = self.__class__.__name__.lower()
-        self.name = name
 
-        logging = kwargs.get('logging', False)
-        self.logging = logging
 
-        self.vars = {}
-        self.placeholders = {}
 
-        self.layers = []
-        self.activations = []
-
-        self.inputs = None
-        self.outputs = None
-
-        self.loss = 0
-        self.accuracy = 0
-        self.optimizer = None
-        self.opt_op = None
-
-    def _build(self):
-        raise NotImplementedError
-
-    def build(self):
-        """
-        Wrapper for _build()
-        """
-        with tf.variable_scope(self.name):
-            self._build()
-
-        # Build sequential layer model
-        self.activations.append(self.inputs)
-        for layer in self.layers:
-            hidden = layer(self.activations[-1])
-            self.activations.append(hidden)
-        self.outputs = self.activations[-1]
-
-        # Store model variables for easy access
-        variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.name)
-        self.vars = {var.name: var for var in variables}
-
-        # Build metrics
-        self._loss()
-        self._accuracy()
-
-        self.opt_op = self.optimizer.minimize(self.loss)
-
-    def predict(self):
-        pass
-
-    def _loss(self):
-        raise NotImplementedError
-
-    def _accuracy(self):
-        raise NotImplementedError
-
-    def save(self, sess=None):
-        if not sess:
-            raise AttributeError("TensorFlow session not provided.")
-        saver = tf.train.Saver(self.vars)
-        save_path = saver.save(sess, "tmp/%s.ckpt" % self.name)
-        print("Model saved in file: %s" % save_path)
-
-    def load(self, sess=None):
-        if not sess:
-            raise AttributeError("TensorFlow session not provided.")
-        saver = tf.train.Saver(self.vars)
-        save_path = "tmp/%s.ckpt" % self.name
-        saver.restore(sess, save_path)
-        print("Model restored from file: %s" % save_path)
-
-
-class MLP(Model):
+class MLP(keras.Model):
     def __init__(self, placeholders, input_dim, **kwargs):
         super(MLP, self).__init__(**kwargs)
 
@@ -148,19 +72,16 @@ def __init__(self, input_dim, output_dim, num_features_nonzero, **kwargs):
                                             output_dim=args.hidden1, # 16
                                             num_features_nonzero=num_features_nonzero,
                                             activation=tf.nn.relu,
-                                            dropout=0.5,
+                                            dropout=args.dropout,
                                             is_sparse_inputs=True))
 
         self.layers_.append(GraphConvolution(input_dim=args.hidden1, # 16
                                             output_dim=self.output_dim, # 7
                                             num_features_nonzero=num_features_nonzero,
                                             activation=lambda x: x,
-                                            dropout=0.5))
+                                            dropout=args.dropout))
 
 
-    def _accuracy(self):
-        self.accuracy = masked_accuracy(self.outputs, self.placeholders['labels'],
-                                        self.placeholders['labels_mask'])
 
 
     def call(self, inputs, training=None):
@@ -187,10 +108,11 @@ def call(self, inputs, training=None):
         # Cross entropy error
         loss += masked_softmax_cross_entropy(output, label, mask)
 
-
         acc = masked_accuracy(output, label, mask)
 
         return loss, acc
 
+
+
     def predict(self):
         return tf.nn.softmax(self.outputs)

train.py

@@ -9,15 +9,18 @@
 import  os
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 print('tf version:', tf.__version__)
+assert tf.__version__.startswith('2.')
 
-# Set random seed
+
+
+# set random seed
 seed = 123
 np.random.seed(seed)
 tf.random.set_seed(seed)
 
 
 
-# Load data
+# load data
 adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(args.dataset)
 print('adj:', adj.shape)
 print('features:', features.shape)
@@ -26,7 +29,6 @@
 
 
 
-# Some preprocessing
 # D^-1@X
 features = preprocess_features(features) # [49216, 2], [49216], [2708, 1433]
 print('features coordinates::', features[0].shape)
@@ -49,19 +51,10 @@
 else:
     raise ValueError('Invalid argument for model: ' + str(args.model))
 
-# # Define placeholders
-# placeholders = {
-#     'support': [tf.sparse_placeholder(tf.float32) for _ in range(num_supports)],
-#     'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)),
-#     'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),
-#     'labels_mask': tf.placeholder(tf.int32),
-#     'dropout': tf.placeholder_with_default(0., shape=()),
-#     'num_features_nonzero': tf.placeholder(tf.int32)  # helper variable for sparse dropout
-# }
+
 
 # Create model
-model = GCN(input_dim=features[2][1], output_dim=y_train.shape[1],
-                num_features_nonzero=features[1].shape) # [1433]
+model = GCN(input_dim=features[2][1], output_dim=y_train.shape[1], num_features_nonzero=features[1].shape) # [1433]
 
 
 
@@ -74,26 +67,22 @@
 support = [tf.cast(tf.SparseTensor(*support[0]), dtype=tf.float32)]
 num_features_nonzero = features.values.shape
 dropout = args.dropout
-print(num_features_nonzero, support[0].dtype)
 
 
-optimizer = optimizers.Adam(lr=1e-3)
+optimizer = optimizers.Adam(lr=1e-2)
 
 cost_val = []
 
-# Train model
-for epoch in range(args.epochs):
 
-    t = time.time()
+for epoch in range(args.epochs):
 
     with tf.GradientTape() as tape:
         loss, acc = model((features, train_label, train_mask,support))
     grads = tape.gradient(loss, model.trainable_variables)
     optimizer.apply_gradients(zip(grads, model.trainable_variables))
 
 
-
-    _, val_acc = model((features, val_label, val_mask, support))
+    _, val_acc = model((features, val_label, val_mask, support), training=False)
 
 
     print(epoch, float(loss), float(acc), '\tval:', float(val_acc))

utils.py

@@ -1,21 +1,25 @@
-import numpy as np
-import pickle as pkl
-import networkx as nx
-import scipy.sparse as sp
-from scipy.sparse.linalg.eigen.arpack import eigsh
-import sys
+import  numpy as np
+import  pickle as pkl
+import  networkx as nx
+import  scipy.sparse as sp
+from    scipy.sparse.linalg.eigen.arpack import eigsh
+import  sys
 
 
 def parse_index_file(filename):
-    """Parse index file."""
+    """
+    Parse index file.
+    """
     index = []
     for line in open(filename):
         index.append(int(line.strip()))
     return index
 
 
 def sample_mask(idx, l):
-    """Create mask."""
+    """
+    Create mask.
+    """
     mask = np.zeros(l)
     mask[idx] = 1
     return np.array(mask, dtype=np.bool)
@@ -139,19 +143,13 @@ def preprocess_adj(adj):
     return sparse_to_tuple(adj_normalized)
 
 
-def construct_feed_dict(features, support, labels, labels_mask, placeholders):
-    """Construct feed dictionary."""
-    feed_dict = dict()
-    feed_dict.update({placeholders['labels']: labels})
-    feed_dict.update({placeholders['labels_mask']: labels_mask})
-    feed_dict.update({placeholders['features']: features})
-    feed_dict.update({placeholders['support'][i]: support[i] for i in range(len(support))})
-    feed_dict.update({placeholders['num_features_nonzero']: features[1].shape})
-    return feed_dict
+
 
 
 def chebyshev_polynomials(adj, k):
-    """Calculate Chebyshev polynomials up to order k. Return a list of sparse matrices (tuple representation)."""
+    """
+    Calculate Chebyshev polynomials up to order k. Return a list of sparse matrices (tuple representation).
+    """
     print("Calculating Chebyshev polynomials up to order {}...".format(k))
 
     adj_normalized = normalize_adj(adj)