Tensorflow多線程圖像加載

Question

所以我有這個玩具的例子代碼;

import glob 
from tqdm import tqdm 
import tensorflow as tf 

imgPaths = glob.glob("/home/msmith/imgs/*/*") # Some images 

filenameQ = tf.train.string_input_producer(imgPaths) 
reader = tf.WholeFileReader() 
key, value = reader.read(filenameQ) 

img = tf.image.decode_jpeg(value) 
init_op = tf.initialize_all_variables() 

with tf.Session() as sess: 
    sess.run(init_op) 
    coord = tf.train.Coordinator() 
    threads = tf.train.start_queue_runners(coord=coord) 
    for i in tqdm(range(10000)): 
     img.eval().mean() 

加載圖像並打印每個圖像的均值。如何對其進行編輯，以便對圖像的加載部分進行多線程處理，這是目前我的tf圖像腳本的瓶頸。

Answer 1

編輯（2018/3/5）：使用tf.data API現在更容易獲得相同的結果。

import glob 
from tqdm import tqdm 
import tensorflow as tf 

imgPaths = glob.glob("/home/msmith/imgs/*/*") # Some images 

dataset = (tf.data.Dataset.from_tensor_slices(imgPaths) 
      .map(lambda x: tf.reduce_mean(tf.decode_jpeg(tf.read_file(x))), 
       num_parallel_calls=16) 
      .prefetch(128)) 

iterator = dataset.make_one_shot_iterator() 
next_mean = iterator.get_next() 

with tf.Session() as sess: 
    for i in tqdm(range(10000)): 
     sess.run(next_mean) 

---

如sygi表明在their comment，一個tf.train.QueueRunner可以用於定義在一個單獨的線程運行一些OPS，和（通常）排隊值成TensorFlow隊列。

import glob 
from tqdm import tqdm 
import tensorflow as tf 

imgPaths = glob.glob("/home/msmith/imgs/*/*") # Some images 

filenameQ = tf.train.string_input_producer(imgPaths) 

# Define a subgraph that takes a filename, reads the file, decodes it, and                      
# enqueues it.                                     
filename = filenameQ.dequeue() 
image_bytes = tf.read_file(filename) 
decoded_image = tf.image.decode_jpeg(image_bytes) 
image_queue = tf.FIFOQueue(128, [tf.uint8], None) 
enqueue_op = image_queue.enqueue(decoded_image) 

# Create a queue runner that will enqueue decoded images into `image_queue`.                     
NUM_THREADS = 16 
queue_runner = tf.train.QueueRunner(
    image_queue, 
    [enqueue_op] * NUM_THREADS, # Each element will be run from a separate thread.                      
    image_queue.close(), 
    image_queue.close(cancel_pending_enqueues=True)) 

# Ensure that the queue runner threads are started when we call                        
# `tf.train.start_queue_runners()` below.                              
tf.train.add_queue_runner(queue_runner) 

# Dequeue the next image from the queue, for returning to the client.                       
img = image_queue.dequeue() 

init_op = tf.global_variables_initializer() 

with tf.Session() as sess: 
    sess.run(init_op) 
    coord = tf.train.Coordinator() 
    threads = tf.train.start_queue_runners(sess=sess, coord=coord) 
    for i in tqdm(range(10000)): 
     img.eval().mean()