hezi-resheff / oreilly-learning-tensorflow Goto Github PK

Hi, first of all, great book !
Quick question: in chapter 5:

with tf.name_scope('linear_layer_weights') as scope:
    # Iterate across time, apply linear layer to all RNN outputs
    all_outputs = tf.map_fn(get_linear_layer, all_hidden_states)
    # Get last output
    output = all_outputs[-1]

Why do we calculate "all_outputs", since we are interested only by the last one ?
I'm not worried about optimization, just trying to make sure I understand well the process.

Thanks !

Could you please upload the code for CIFAR10 outlined in Chapter 4?

Is there other missing code? I am reading through the book, and having the code on Git is really helpful.

I wrote the following code from the book ,

import tensorflow as tf
import os
import cPickle
import numpy as np

def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)

def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)

def conv2d(x,W):
return tf.nn.conv2d(x, W, strides=[1,1,1,1],padding='SAME')

def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME')

def conv_layer(input, shape):
W = weight_variable(shape)
b = bias_variable([shape[3]])
return tf.nn.relu(conv2d(input,W)+b)

def full_layer(input, size):
in_size = int(input.get_shape()[1])
W = weight_variable([in_size, size])
b = bias_variable([size])
return tf.matmul(input, W)+b

class CifarLoader(object):
def init(self, source_files):
self._source = source_files
self._i = 0
self.images = None
self.labels = None

    def     load(self):
            data    =       [unpickle(f) for f in self._source]
            images  =       np.vstack([d["data"] for d in data])
            n       =       len(images)
            self.images     =       images.reshape(n,3,32,32).transpose(0,2,3,1).astype(float)/255
            self.labels     =       one_hot(np.hstack([d["labels"] for d in data]), 10)
            return self

    def     next_batch(self, batch_size):
            x,y     =       self.images[self._i:self._i+batch_size],self.labels[self._i:self._i+batch_size]
            self._i =       (self._i+batch_size)%len(self.images)
            return  x,y

def unpickle(file):
with open(os.path.join(DATA_PATH, file), 'rb') as fo:
dict = cPickle.load(fo)
return dict

def one_hot(vec, vals=10):
n = len(vec)
out = np.zeros((n,vals))
out[range(n), vec]=1
return out

class CifarDataManager(object):
def init(self):
self.train = CifarLoader(["data_batch_{}".format(i) for i in range(1,6)]).load()
self.test = CifarLoader(["test_batch"]).load()

DATA_PATH ="/home/amalik/Programming/Tensorflow/chapter4/CIFAR10/cifar-10-batches-py"

cifar = CifarDataManager()

x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])

y_ = tf.placeholder(tf.float32, shape=[None, 10])

keep_prob = tf.placeholder(tf.float32)

conv1 = conv_layer(x, shape=[5,5,3,32])
conv1_pool= max_pool_2x2(conv1)

conv2 = conv_layer(conv1_pool, shape=[5,5,32,64])
conv2_pool = max_pool_2x2(conv2)
conv2_flate = tf.reshape(conv2_pool, [-1,8864])

full_1 = tf.nn.relu(full_layer(conv2_flate, 1024))
full1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob)

y_conv = full_layer(full1_drop, 10)

cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction= tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))

accueacy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

STEPS=10
BATCH_SIZE=1

def test(sess):
X = cifar.test.images.reshape(10, 1000, 32, 32, 3)
Y = cifar.test.labels.reshape(10, 1000, 10)
acc = np.mean([sess.run(accuracy, feed_dict={x:X[i], y_:Y[i], keep_prob:1.0}) for i in range(10)])
print "Accuracy : {:.4}%".format(acc*100)

with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(STEPS):
batch = cifar.train.next_batch(BATCH_SIZE)
sess.run(train_step, feed_dict={x:batch[0], y_:batch[1], keep_prob:0.5})
test(sess)

However, I am getting the following error:
python cnn.py
2018-05-29 14:42:48.295115: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX
2018-05-29 14:42:48.520456: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Found device 0 with properties:
name: Tesla K20Xm major: 3 minor: 5 memoryClockRate(GHz): 0.732
pciBusID: 0000:08:00.0
totalMemory: 5.57GiB freeMemory: 152.44MiB
2018-05-29 14:42:48.520564: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1120] Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: Tesla K20Xm, pci bus id: 0000:08:00.0, compute capability: 3.5)
2018-05-29 14:42:48.536010: E tensorflow/stream_executor/cuda/cuda_driver.cc:936] failed to allocate 152.44M (159842304 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY
2018-05-29 14:42:49.618407: E tensorflow/stream_executor/cuda/cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-05-29 14:42:49.634824: E tensorflow/stream_executor/cuda/cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-05-29 14:42:49.649660: E tensorflow/stream_executor/cuda/cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-05-29 14:42:49.662774: E tensorflow/stream_executor/cuda/cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-05-29 14:42:49.675470: E tensorflow/stream_executor/cuda/cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-05-29 14:42:49.687850: E tensorflow/stream_executor/cuda/cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-05-29 14:42:50.614900: E tensorflow/stream_executor/cuda/cuda_dnn.cc:385] could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR
2018-05-29 14:42:50.614959: E tensorflow/stream_executor/cuda/cuda_dnn.cc:352] could not destroy cudnn handle: CUDNN_STATUS_BAD_PARAM
2018-05-29 14:42:50.614979: F tensorflow/core/kernels/conv_ops.cc:667] Check failed: stream->parent()->GetConvolveAlgorithms( conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)

I am using a very small batchs size.

argument 'global_step = global_step' should be inserted at 'minimize()'.
Only in this way, global_step increments one by one every training iteration and weights decay

# Learning rate decay
global_step = tf.Variable(0, trainable=False)
learningRate = tf.train.exponential_decay(learning_rate=0.1,
                                          global_step=global_step,
                                          decay_steps=1000,
                                          decay_rate=0.95,
                                          staircase=True)
train_step = tf.train.GradientDescentOptimizer(learningRate).minimize(loss, global_step= global_step)

I am using Ubuntu 18.04 and Tensorflow 1.13.1! I installed "layers" via pip.
But if I run the Code it doesn't work!

ModuleNotFoundError: No module named 'layers'

Getting the following error when I run vanilla_rnn_with_tfboard.py

Traceback (most recent call last):
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 1322, in _do_call
    return fn(*args)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 1307, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 1409, in _call_tf_sessionrun
    run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(128, 128), b.shape=(128, 128), m=128, n=128, k=128
	 [[Node: states/while/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](states/while/Switch_2:1, states/while/MatMul/Enter)]]
	 [[Node: linear_layer_weights_1/strided_slice/_21 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_460_linear_layer_weights_1/strided_slice", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "D:/tensorflow/vanilla_rnn_with_tfboard.py", line 127, in <module>
    summary, _ = sess.run([merged, train_step], feed_dict={_inputs: batch_x, y: batch_y})
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 900, in run
    run_metadata_ptr)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 1135, in _run
    feed_dict_tensor, options, run_metadata)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 1316, in _do_run
    run_metadata)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 1335, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(128, 128), b.shape=(128, 128), m=128, n=128, k=128
	 [[Node: states/while/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](states/while/Switch_2:1, states/while/MatMul/Enter)]]
	 [[Node: linear_layer_weights_1/strided_slice/_21 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_460_linear_layer_weights_1/strided_slice", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]

Caused by op 'states/while/MatMul', defined at:
  File "D:/tensorflow/vanilla_rnn_with_tfboard.py", line 67, in <module>
    all_hidden_states = tf.scan(rnn_step, processed_input, initializer=initial_hidden, name='states')
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\functional_ops.py", line 630, in scan
    maximum_iterations=n)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3224, in while_loop
    result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2956, in BuildLoop
    pred, body, original_loop_vars, loop_vars, shape_invariants)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2893, in _BuildLoop
    body_result = body(*packed_vars_for_body)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3194, in <lambda>
    body = lambda i, lv: (i + 1, orig_body(*lv))
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\functional_ops.py", line 619, in compute
    a_out = fn(packed_a, packed_elems)
  File "D:/tensorflow/vanilla_rnn_with_tfboard.py", line 54, in rnn_step
    tf.matmul(x, Wx) + b_rnn)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\math_ops.py", line 2122, in matmul
    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 4278, in mat_mul
    name=name)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
    op_def=op_def)
  File "C:\Users\quartermaine\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InternalError (see above for traceback): Blas GEMM launch failed : a.shape=(128, 128), b.shape=(128, 128), m=128, n=128, k=128
	 [[Node: states/while/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](states/while/Switch_2:1, states/while/MatMul/Enter)]]
	 [[Node: linear_layer_weights_1/strided_slice/_21 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_460_linear_layer_weights_1/strided_slice", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]

chapter2/softmax.py doesn't seem to work with Tensorflow 1.0:

cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_pred, y_true))

File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/nn_ops.py", line 1578, in softmax_cross_entropy_with_logits
labels, logits)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/nn_ops.py", line 1533, in _ensure_xent_args
"named arguments (labels=..., logits=..., ...)" % name)
ValueError: Only call softmax_cross_entropy_with_logits with named arguments (labels=..., logits=..., ...)

• Run the notebooks for Chapters 7 and 10 similar to how Chapter 3's notebooks was run so that people can see the expected output.
• Some of the constants are defined in cells that they do not relate to and thus in the cells that they are for there is missing information. This makes it hard to understand the code since some of the data is missing from the cells related to specific tasks.
• Side Note: Convert some of the existing Python files to Jupyter Notebooks to make it easier for someone to look at the code and see the output of different sections similar to how it is done in the book.

It seems there's a bug in lines 39-40: even sentences are building from the odd ones.

even_sentences.append(" ".join([digit_to_word_map[r] for r in rand_odd_ints])) odd_sentences.append(" ".join([digit_to_word_map[r] for r in rand_even_ints]))

in chapter - 5 an example has been given to show how multiple LSTM blocks can be stacked in order to create RNN classifier with multiple layers. But that code is throwing an error which I failed to debug so far. The error is matrix dimension mismatch [128, 64] matrix multiplication with [96, 128]
The detailed error message is as follows:
Code Blocks:
num_LSTM_layers = 2
with tf.variable_scope("lstm"):

lstm_cell = tf.contrib.rnn.BasicLSTMCell(hidden_layer_size,
                                         forget_bias=1.0)
cell = tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell]*num_LSTM_layers,
                                   state_is_tuple=True)
outputs, states = tf.nn.dynamic_rnn(cell, embed,
                                    sequence_length = _seqlens,
                                    dtype=tf.float32)

Error:

InvalidArgumentError Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py in _call_cpp_shape_fn_impl(op, input_tensors_needed, input_tensors_as_shapes_needed, require_shape_fn)
685 graph_def_version, node_def_str, input_shapes, input_tensors,
--> 686 input_tensors_as_shapes, status)
687 except errors.InvalidArgumentError as err:

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in exit(self, type_arg, value_arg, traceback_arg)
472 compat.as_text(c_api.TF_Message(self.status.status)),
--> 473 c_api.TF_GetCode(self.status.status))
474 # Delete the underlying status object from memory otherwise it stays alive

InvalidArgumentError: Dimensions must be equal, but are 64 and 96 for 'lstm_1/rnn/while/rnn/multi_rnn_cell/cell_0/cell_0/basic_lstm_cell/MatMul_1' (op: 'MatMul') with input shapes: [128,64], [96,128].

During handling of the above exception, another exception occurred:

ValueError Traceback (most recent call last)
in ()
8 outputs, states = tf.nn.dynamic_rnn(cell, embed,
9 sequence_length = _seqlens,
---> 10 dtype=tf.float32)

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py in dynamic_rnn(cell, inputs, sequence_length, initial_state, dtype, parallel_iterations, swap_memory, time_major, scope)
612 swap_memory=swap_memory,
613 sequence_length=sequence_length,
--> 614 dtype=dtype)
615
616 # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py in _dynamic_rnn_loop(cell, inputs, initial_state, parallel_iterations, swap_memory, sequence_length, dtype)
775 loop_vars=(time, output_ta, state),
776 parallel_iterations=parallel_iterations,
--> 777 swap_memory=swap_memory)
778
779 # Unpack final output if not using output tuples.

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py in while_loop(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name)
2814 loop_context = WhileContext(parallel_iterations, back_prop, swap_memory) # pylint: disable=redefined-outer-name
2815 ops.add_to_collection(ops.GraphKeys.WHILE_CONTEXT, loop_context)
-> 2816 result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
2817 return result
2818

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py in BuildLoop(self, pred, body, loop_vars, shape_invariants)
2638 self.Enter()
2639 original_body_result, exit_vars = self._BuildLoop(
-> 2640 pred, body, original_loop_vars, loop_vars, shape_invariants)
2641 finally:
2642 self.Exit()

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py in _BuildLoop(self, pred, body, original_loop_vars, loop_vars, shape_invariants)
2588 structure=original_loop_vars,
2589 flat_sequence=vars_for_body_with_tensor_arrays)
-> 2590 body_result = body(*packed_vars_for_body)
2591 if not nest.is_sequence(body_result):
2592 body_result = [body_result]

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py in _time_step(time, output_ta_t, state)
758 call_cell=call_cell,
759 state_size=state_size,
--> 760 skip_conditionals=True)
761 else:
762 (output, new_state) = call_cell()

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py in _rnn_step(time, sequence_length, min_sequence_length, max_sequence_length, zero_output, state, call_cell, state_size, skip_conditionals)
234 # steps. This is faster when max_seq_len is equal to the number of unrolls
235 # (which is typical for dynamic_rnn).
--> 236 new_output, new_state = call_cell()
237 nest.assert_same_structure(state, new_state)
238 new_state = nest.flatten(new_state)

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py in ()
746
747 input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t)
--> 748 call_cell = lambda: cell(input_t, state)
749
750 if sequence_length is not None:

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py in call(self, inputs, state, scope)
181 with vs.variable_scope(vs.get_variable_scope(),
182 custom_getter=self._rnn_get_variable):
--> 183 return super(RNNCell, self).call(inputs, state)
184
185 def _rnn_get_variable(self, getter, *args, **kwargs):

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/layers/base.py in call(self, inputs, *args, **kwargs)
573 if in_graph_mode:
574 self._assert_input_compatibility(inputs)
--> 575 outputs = self.call(inputs, *args, **kwargs)
576
577 if outputs is None:

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py in call(self, inputs, state)
1064 [-1, cell.state_size])
1065 cur_state_pos += cell.state_size
-> 1066 cur_inp, new_state = cell(cur_inp, cur_state)
1067 new_states.append(new_state)
1068

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py in call(self, inputs, state, scope)
181 with vs.variable_scope(vs.get_variable_scope(),
182 custom_getter=self._rnn_get_variable):
--> 183 return super(RNNCell, self).call(inputs, state)
184
185 def _rnn_get_variable(self, getter, *args, **kwargs):

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/layers/base.py in call(self, inputs, *args, **kwargs)
573 if in_graph_mode:
574 self._assert_input_compatibility(inputs)
--> 575 outputs = self.call(inputs, *args, **kwargs)
576
577 if outputs is None:

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py in call(self, inputs, state)
439 # i = input_gate, j = new_input, f = forget_gate, o = output_gate
440 i, j, f, o = array_ops.split(
--> 441 value=self._linear([inputs, h]), num_or_size_splits=4, axis=1)
442
443 new_c = (

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py in call(self, args)
1187 res = math_ops.matmul(args[0], self._weights)
1188 else:
-> 1189 res = math_ops.matmul(array_ops.concat(args, 1), self._weights)
1190 if self._build_bias:
1191 res = nn_ops.bias_add(res, self._biases)

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py in matmul(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name)
1889 else:
1890 return gen_math_ops._mat_mul(
-> 1891 a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
1892
1893

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py in _mat_mul(a, b, transpose_a, transpose_b, name)
2435 _, _, _op = _op_def_lib._apply_op_helper(
2436 "MatMul", a=a, b=b, transpose_a=transpose_a, transpose_b=transpose_b,
-> 2437 name=name)
2438 _result = _op.outputs[:]
2439 _inputs_flat = _op.inputs

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
785 op = g.create_op(op_type_name, inputs, output_types, name=scope,
786 input_types=input_types, attrs=attr_protos,
--> 787 op_def=op_def)
788 return output_structure, op_def.is_stateful, op
789

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in create_op(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_shapes, compute_device)
2956 op_def=op_def)
2957 if compute_shapes:
-> 2958 set_shapes_for_outputs(ret)
2959 self._add_op(ret)
2960 self._record_op_seen_by_control_dependencies(ret)

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in set_shapes_for_outputs(op)
2207 shape_func = _call_cpp_shape_fn_and_require_op
2208
-> 2209 shapes = shape_func(op)
2210 if shapes is None:
2211 raise RuntimeError(

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in call_with_requiring(op)
2157
2158 def call_with_requiring(op):
-> 2159 return call_cpp_shape_fn(op, require_shape_fn=True)
2160
2161 _call_cpp_shape_fn_and_require_op = call_with_requiring

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py in call_cpp_shape_fn(op, require_shape_fn)
625 res = _call_cpp_shape_fn_impl(op, input_tensors_needed,
626 input_tensors_as_shapes_needed,
--> 627 require_shape_fn)
628 if not isinstance(res, dict):
629 # Handles the case where _call_cpp_shape_fn_impl calls unknown_shape(op).

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py in _call_cpp_shape_fn_impl(op, input_tensors_needed, input_tensors_as_shapes_needed, require_shape_fn)
689 missing_shape_fn = True
690 else:
--> 691 raise ValueError(err.message)
692
693 if missing_shape_fn:

ValueError: Dimensions must be equal, but are 64 and 96 for 'lstm_1/rnn/while/rnn/multi_rnn_cell/cell_0/cell_0/basic_lstm_cell/MatMul_1' (op: 'MatMul') with input shapes: [128,64], [96,128].

Hello author, when will you committee Chapter 5&6?

loss = tf.reduce_mean(
tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,
negative_samples, vocabulary_size))

This line returns the following error.
ValueError: Tensor conversion requested dtype int32 for Tensor with dtype float32: 'Tensor("nce_loss/Reshape_1:0", shape=(?, 1, ?), dtype=float32)'

Hi,

This is not an issue. But I'm having hard time in the usage of time_steps and element_size. In case of MNIST image, each input size is 28 X 28. So the author is saying element_size is 28 columns (considering iteration is going over the rows) and time_steps is 28 rows?

If that's the case, why in text classification we are not using element_size dimension in placeholder? Will it not throw an error?

Lastly, in labels for MNIST, we are not considering the batch_size. But for test classification, we are doing it. Why that so?

It will be a great help if you help me answer these questions? I'm struggling for a long time to get the answer. In all blogs and other references, the codes are exactly same as the book. But no one is reasoning why this difference is.

Thank you!

Which chapters are still valid considering tensorflow 2.x?
Regards
Ivan

Hi, I have learned deep learning & machine learning with provided codes.
However, when i studied the convolutional_neural_network mnist example(code name mnist_cnn ) in chapter 4, occur the same accuracy. is it possible to occur the same accuracy? Likewise, i start again the mnist example, it also occurs training accuracy same to prior practiced.

Running python 3.6, the MNIST example from chapter 2 gives more warning than actual output:

(anns) jimhigson@budapest ~/d/A/Learning Tensorflow Book> python Oreilly-Learning-TensorFlow/02__up_and_running/softmax.py
WARNING:tensorflow:From Oreilly-Learning-TensorFlow/02__up_and_running/softmax.py:10: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use alternatives such as official/mnist/dataset.py from tensorflow/models.
WARNING:tensorflow:From /Users/jimhigson/dev/ANNs/Learning Tensorflow Book/envs/anns/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.
Instructions for updating:
Please write your own downloading logic.
WARNING:tensorflow:From /Users/jimhigson/dev/ANNs/Learning Tensorflow Book/envs/anns/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist)is deprecated and will be removed in a future version.
Instructions for updating:
Please use tf.data to implement this functionality.
Extracting /tmp/data/train-images-idx3-ubyte.gz
WARNING:tensorflow:From /Users/jimhigson/dev/ANNs/Learning Tensorflow Book/envs/anns/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist)is deprecated and will be removed in a future version.
Instructions for updating:
Please use tf.data to implement this functionality.
Extracting /tmp/data/train-labels-idx1-ubyte.gz
WARNING:tensorflow:From /Users/jimhigson/dev/ANNs/Learning Tensorflow Book/envs/anns/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use tf.one_hot on tensors.
Extracting /tmp/data/t10k-images-idx3-ubyte.gz
Extracting /tmp/data/t10k-labels-idx1-ubyte.gz
WARNING:tensorflow:From /Users/jimhigson/dev/ANNs/Learning Tensorflow Book/envs/anns/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
Instructions for updating:
Please use alternatives such as official/mnist/dataset.py from tensorflow/models.
WARNING:tensorflow:From Oreilly-Learning-TensorFlow/02__up_and_running/softmax.py:19: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See `tf.nn.softmax_cross_entropy_with_logits_v2`.

2018-10-07 13:09:56.778201: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
Accuracy: 90.97%

It would be good if there were an updated version in the git repo, explaining what the differences and why the old ways were deprecated.

I am currently going through your book "Learning TensorFlow: A Guide to Building Deep Learning Systems". It is now a bit older book and uses tensorflow 1.x version and the mnist dataset from tensorflow.examples.tutorials.mnist. Since I have the latest version of tensorflow installed, I have tried to modify the code from Chapter 4 (mnist_cnn.py) and I am close to getting it to run but I have an issue with loading the mnist correctly for training. Here is my code modified code:

import tensorflow_datasets as tfds
import tensorflow as tf
import numpy as np

from layers import conv_layer, max_pool_2x2, full_layer

tf.compat.v1.disable_eager_execution()

DATA_DIR = '../data/'
MINIBATCH_SIZE = 50
STEPS = 5000


mnist = tfds.load(name='mnist', split=['train', 'test'], data_dir=DATA_DIR)

x = tf.compat.v1.placeholder(tf.float32, shape=[None, 784])
y_ = tf.compat.v1.placeholder(tf.float32, shape=[None, 10])

x_image = tf.reshape(x, [-1, 28, 28, 1])
conv1 = conv_layer(x_image, shape=[5, 5, 1, 32])
conv1_pool = max_pool_2x2(conv1)

conv2 = conv_layer(conv1_pool, shape=[5, 5, 32, 64])
conv2_pool = max_pool_2x2(conv2)

conv2_flat = tf.reshape(conv2_pool, [-1, 7*7*64])
full_1 = tf.nn.relu(full_layer(conv2_flat, 1024))

keep_prob = tf.compat.v1.placeholder(tf.float32)
full1_drop = tf.nn.dropout(full_1, rate=1-keep_prob)

y_conv = full_layer(full1_drop, 10)

cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))
train_step = tf.compat.v1.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

with tf.compat.v1.Session() as sess:
    sess.run(tf.compat.v1.global_variables_initializer())

    for i in range(STEPS):
        batch = mnist.train.next_batch(MINIBATCH_SIZE)

        if i % 100 == 0:
            train_accuracy = sess.run(accuracy, feed_dict={x: batch[0], y_: batch[1],
                                                           keep_prob: 1.0})
            print("step {}, training accuracy {}".format(i, train_accuracy))

        sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})

    X = mnist.test.images.reshape(10, 1000, 784)
    Y = mnist.test.labels.reshape(10, 1000, 10)
    test_accuracy = np.mean(
        [sess.run(accuracy, feed_dict={x: X[i], y_: Y[i], keep_prob: 1.0}) for i in range(10)])

print("test accuracy: {}".format(test_accuracy))

When I run my modified code I get this error:

File "/home/af/Dokumenter/Programs/LearningTensorFlow/Chapter 4/mnist_cnn.py", line 43, in <module>
    batch = mnist.train.next_batch(MINIBATCH_SIZE)
AttributeError: 'list' object has no attribute 'train'

I am unsure how to modify that line now that I am loading the mnist dataset from tensorflow_datasets now. Any advice or hints would be welcome.

When I run tf_records_end_to_end.py, it hangs in line 101 sess.run([train_op])