详解视频中动作识别模型与代码实践

import cv2 import os # 视频数据集存储位置 video_path = './dataset_subset/' # 生成的图像数据集存储位置 save_path = './dataset/' # 如果文件路径不存在则创建路径 if not os.path.exists(save_path):  os.mkdir(save_path) # 获取动作列表 action_list = os.listdir(video_path) # 遍历所有动作 for action in action_list:  if action.startswith(".")==False:  if not os.path.exists(save_path+action):  os.mkdir(save_path+action)  video_list = os.listdir(video_path+action)  # 遍历所有视频  for video in video_list:             prefix = video.split('.')[0]  if not os.path.exists(os.path.join(save_path, action, prefix)):  os.mkdir(os.path.join(save_path, action, prefix))  save_name = os.path.join(save_path, action, prefix) + '/'  video_name = video_path+action+'/'+video  # 读取视频文件  # cap为视频的帧             cap = cv2.VideoCapture(video_name)  # fps为帧率             fps = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))  fps_count = 0  for i in range(fps):                 ret, frame = cap.read()  if ret:  # 将帧画面写入图片文件中                     cv2.imwrite(save_name+str(10000+fps_count)+'.jpg',frame)  fps_count += 1

from keras.layers import Dense,Dropout,Conv3D,Input,MaxPool3D,Flatten,Activation, ZeroPadding3D from keras.regularizers import l2 from keras.models import Model, Sequential # 输入数据为 112×112 的图片，16帧， 3通道 input_shape = (112,112,16,3) # 权重衰减率 weight_decay = 0.005 # 类型数量，我们使用UCF-101 为数据集，所以为101 nb_classes = 101 # 构建模型结构 inputs = Input(input_shape) x = Conv3D(64,(3,3,3),strides=(1,1,1),padding='same',            activation='relu',kernel_regularizer=l2(weight_decay))(inputs) x = MaxPool3D((2,2,1),strides=(2,2,1),padding='same')(x) x = Conv3D(128,(3,3,3),strides=(1,1,1),padding='same',            activation='relu',kernel_regularizer=l2(weight_decay))(x) x = MaxPool3D((2,2,2),strides=(2,2,2),padding='same')(x) x = Conv3D(128,(3,3,3),strides=(1,1,1),padding='same',            activation='relu',kernel_regularizer=l2(weight_decay))(x) x = MaxPool3D((2,2,2),strides=(2,2,2),padding='same')(x) x = Conv3D(256,(3,3,3),strides=(1,1,1),padding='same',            activation='relu',kernel_regularizer=l2(weight_decay))(x) x = MaxPool3D((2,2,2),strides=(2,2,2),padding='same')(x) x = Conv3D(256, (3, 3, 3), strides=(1, 1, 1), padding='same',            activation='relu',kernel_regularizer=l2(weight_decay))(x) x = MaxPool3D((2, 2, 2), strides=(2, 2, 2), padding='same')(x) x = Flatten()(x) x = Dense(2048,activation='relu',kernel_regularizer=l2(weight_decay))(x) x = Dropout(0.5)(x) x = Dense(2048,activation='relu',kernel_regularizer=l2(weight_decay))(x) x = Dropout(0.5)(x) x = Dense(nb_classes,kernel_regularizer=l2(weight_decay))(x) x = Activation('softmax')(x) model = Model(inputs, x) Using TensorFlow backend. /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.   _np_qint8 = np.dtype([("qint8", np.int8, 1)]) /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.   _np_quint8 = np.dtype([("quint8", np.uint8, 1)]) /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.   _np_qint16 = np.dtype([("qint16", np.int16, 1)]) /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.   _np_quint16 = np.dtype([("quint16", np.uint16, 1)]) /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.   _np_qint32 = np.dtype([("qint32", np.int32, 1)]) /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.  np_resource = np.dtype([("resource", np.ubyte, 1)]) WARNING:tensorflow:From /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. WARNING:tensorflow:From /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.

model.summary()

model.input <tf.Tensor 'input_1:0' shape=(?, 112, 112, 16, 3) dtype=float32>

# 引用必要的库 from keras.optimizers import SGD,Adam from keras.utils import np_utils import numpy as np import random import cv2 import matplotlib.pyplot as plt # 自定义callbacks from schedules import onetenth_4_8_12 INFO:matplotlib.font_manager:font search path ['/home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf', '/home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/afm', '/home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/pdfcorefonts'] INFO:matplotlib.font_manager:generated new fontManager

img_path = save_path # 图片文件存储位置 results_path = './results' # 训练结果保存位置 if not os.path.exists(results_path):  os.mkdir(results_path)

cates = os.listdir(img_path) train_list = [] test_list = [] # 遍历所有的动作类型 for cate in cates:     videos = os.listdir(os.path.join(img_path, cate))     length = len(videos)//5  # 训练集大小，随机取视频文件加入训练集     train= random.sample(videos, length*4)  train_list.extend(train)  # 将余下的视频加入测试集  for video in videos:  if video not in train:  test_list.append(video) print("训练集为：")  print( train_list) print("共%d 个视频n"%(len(train_list))) print("验证集为：")  print(test_list) print("共%d 个视频"%(len(test_list)))

def process_data(img_path, file_list,batch_size=16,train=True):     batch = np.zeros((batch_size,16,112,112,3),dtype='float32')     labels = np.zeros(batch_size,dtype='int')  cate_list = os.listdir(img_path)  def read_classes():         path = "./classInd.txt"  with open(path, "r+") as f:             lines = f.readlines()         classes = {}  for line in lines:  c_id = line.split()[0]  c_name = line.split()[1]             classes[c_name] =c_id   return classes  classes_dict = read_classes()  for file in file_list:         cate = file.split("_")[1]  img_list = os.listdir(os.path.join(img_path, cate, file))  img_list.sort()  batch_img = []  for i in range(batch_size):             path = os.path.join(img_path, cate, file)             label = int(classes_dict[cate])-1             symbol = len(img_list)//16  if train:  # 随机进行裁剪  crop_x = random.randint(0, 15)  crop_y = random.randint(0, 58)  # 随机进行翻转  is_flip = random.randint(0, 1)  # 以16 帧为单位  for j in range(16):  img = img_list[symbol + j]                     image = cv2.imread( path + '/' + img)                     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)                     image = cv2.resize(image, (171, 128))  if is_flip == 1:                         image = cv2.flip(image, 1)                     batch[i][j][:][:][:] = image[crop_x:crop_x + 112, crop_y:crop_y + 112, :]                     symbol-=1  if symbol<0:  break                 labels[i] = label  else:  for j in range(16):  img = img_list[symbol + j]                     image = cv2.imread( path + '/' + img)                     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)                     image = cv2.resize(image, (171, 128))                     batch[i][j][:][:][:] = image[8:120, 30:142, :]                     symbol-=1  if symbol<0:  break                 labels[i] = label  return batch, labels batch, labels = process_data(img_path, train_list) print("每个batch的形状为：%s"%(str(batch.shape))) print("每个label的形状为：%s"%(str(labels.shape))) 每个batch的形状为：(16, 16, 112, 112, 3) 每个label的形状为：(16,)

def generator_train_batch(train_list, batch_size, num_classes, img_path):  while True:  # 读取一个batch的数据  x_train, x_labels = process_data(img_path, train_list, batch_size=16,train=True)         x = preprocess(x_train)  # 形成input要求的数据格式         y = np_utils.to_categorical(np.array(x_labels), num_classes)         x = np.transpose(x, (0,2,3,1,4))  yield x, y def generator_val_batch(test_list, batch_size, num_classes, img_path):  while True:  # 读取一个batch的数据  y_test,y_labels = process_data(img_path, train_list, batch_size=16,train=False)         x = preprocess(y_test)  # 形成input要求的数据格式         x = np.transpose(x,(0,2,3,1,4))         y = np_utils.to_categorical(np.array(y_labels), num_classes)  yield x, y

def preprocess(inputs):  inputs[..., 0] -= 99.9  inputs[..., 1] -= 92.1  inputs[..., 2] -= 82.6  inputs[..., 0] /= 65.8  inputs[..., 1] /= 62.3  inputs[..., 2] /= 60.3  return inputs # 训练一个epoch大约需4分钟 # 类别数量 num_classes = 101 # batch大小 batch_size = 4 # epoch数量 epochs = 1 # 学习率大小 lr = 0.005 # 优化器定义 sgd = SGD(lr=lr, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # 开始训练 history = model.fit_generator(generator_train_batch(train_list, batch_size, num_classes,img_path),  steps_per_epoch= len(train_list) // batch_size,                               epochs=epochs,                               callbacks=[onetenth_4_8_12(lr)],  validation_data=generator_val_batch(test_list, batch_size,num_classes,img_path),  validation_steps= len(test_list) // batch_size,                               verbose=1) # 对训练结果进行保存 model.save_weights(os.path.join(results_path, 'weights_c3d.h5')) WARNING:tensorflow:From /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.cast instead. Epoch 1/1 20/20 [==============================] - 442s 22s/step - loss: 28.7099 - acc: 0.9344 - val_loss: 27.7600 - val_acc: 1.0000

from IPython.display import clear_output, Image, display, HTML import time import cv2 import base64 import numpy as np

from models import c3d_model model = c3d_model() model.load_weights(os.path.join(results_path, 'weights_c3d.h5'), by_name=True) # 加载刚训练的模型

def arrayShow(img):  _,ret = cv2.imencode('.jpg', img)   return Image(data=ret) 

# 加载所有的类别和编号 with open('./ucfTrainTestlist/classInd.txt', 'r') as f:  class_names = f.readlines()  f.close() # 读取视频文件 video = './videos/v_Punch_g03_c01.avi' cap = cv2.VideoCapture(video) clip = [] # 将视频画面传入模型 while True:  try:  clear_output(wait=True)         ret, frame = cap.read()  if ret:  tmp = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)  clip.append(cv2.resize(tmp, (171, 128)))  # 每16帧进行一次预测  if len(clip) == 16:                 inputs = np.array(clip).astype(np.float32)                 inputs = np.expand_dims(inputs, axis=0)  inputs[..., 0] -= 99.9  inputs[..., 1] -= 92.1  inputs[..., 2] -= 82.6  inputs[..., 0] /= 65.8  inputs[..., 1] /= 62.3  inputs[..., 2] /= 60.3                 inputs = inputs[:,:,8:120,30:142,:]                 inputs = np.transpose(inputs, (0, 2, 3, 1, 4))  # 获得预测结果  pred = model.predict(inputs)                 label = np.argmax(pred[0])  # 将预测结果绘制到画面中                 cv2.putText(frame, class_names[label].split(' ')[-1].strip(), (20, 20),                             cv2.FONT_HERSHEY_SIMPLEX, 0.6,  (0, 0, 255), 1)                 cv2.putText(frame, "prob: %.4f" % pred[0][label], (20, 40),                             cv2.FONT_HERSHEY_SIMPLEX, 0.6,  (0, 0, 255), 1)  clip.pop(0)  # 播放预测后的视频              lines, columns, _ = frame.shape             frame = cv2.resize(frame, (int(columns), int(lines)))  img = arrayShow(frame)             display(img)  time.sleep(0.02)  else:  break  except:  print(0) cap.release()

import numpy as np import tensorflow as tf import i3d WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0. For more information, please see:  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md  * https://github.com/tensorflow/addons If you depend on functionality not listed there, please file an issue.

# 输入图片大小 _IMAGE_SIZE = 224 #  视频的帧数 _SAMPLE_VIDEO_FRAMES = 79 # 输入数据包括两部分：RGB和光流 # RGB和光流数据已经经过提前计算 _SAMPLE_PATHS = {  'rgb': 'data/v_CricketShot_g04_c01_rgb.npy',  'flow': 'data/v_CricketShot_g04_c01_flow.npy', } # 提供了多种可以选择的预训练权重 # 其中，imagenet系列模型从ImageNet的2D权重中拓展而来，其余为视频数据下的预训练权重 _CHECKPOINT_PATHS = {  'rgb': 'data/checkpoints/rgb_scratch/model.ckpt',  'flow': 'data/checkpoints/flow_scratch/model.ckpt',  'rgb_imagenet': 'data/checkpoints/rgb_imagenet/model.ckpt',  'flow_imagenet': 'data/checkpoints/flow_imagenet/model.ckpt', } # 记录类别文件 _LABEL_MAP_PATH = 'data/label_map.txt' # 类别数量为400 NUM_CLASSES = 400

imagenet_pretrained = True # 加载动作类型 kinetics_classes = [x.strip() for x in open(_LABEL_MAP_PATH)] tf.logging.set_verbosity(tf.logging.INFO)

rgb_input = tf.placeholder(tf.float32, shape=(1, _SAMPLE_VIDEO_FRAMES, _IMAGE_SIZE, _IMAGE_SIZE, 3)) with tf.variable_scope('RGB', reuse=tf.AUTO_REUSE):  rgb_model = i3d.InceptionI3d(NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits')  rgb_logits, _ = rgb_model(rgb_input, is_training=False, dropout_keep_prob=1.0) rgb_variable_map = {} for variable in tf.global_variables():  if variable.name.split('/')[0] == 'RGB':  rgb_variable_map[variable.name.replace(':0', '')] = variable rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)

flow_input = tf.placeholder(tf.float32,shape=(1, _SAMPLE_VIDEO_FRAMES, _IMAGE_SIZE, _IMAGE_SIZE, 2)) with tf.variable_scope('Flow', reuse=tf.AUTO_REUSE):  flow_model = i3d.InceptionI3d(NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits')  flow_logits, _ = flow_model(flow_input, is_training=False, dropout_keep_prob=1.0) flow_variable_map = {} for variable in tf.global_variables():  if variable.name.split('/')[0] == 'Flow':  flow_variable_map[variable.name.replace(':0', '')] = variable flow_saver = tf.train.Saver(var_list=flow_variable_map, reshape=True) 

model_logits = rgb_logits + flow_logits model_predictions = tf.nn.softmax(model_logits)

with tf.Session() as sess:  feed_dict = {}  if imagenet_pretrained:  rgb_saver.restore(sess, _CHECKPOINT_PATHS['rgb_imagenet']) # 加载rgb流的模型  else:  rgb_saver.restore(sess, _CHECKPOINT_PATHS['rgb'])  tf.logging.info('RGB checkpoint restored')  if imagenet_pretrained:  flow_saver.restore(sess, _CHECKPOINT_PATHS['flow_imagenet']) # 加载flow流的模型  else:  flow_saver.restore(sess, _CHECKPOINT_PATHS['flow'])  tf.logging.info('Flow checkpoint restored')   start_time = time.time()  rgb_sample = np.load(_SAMPLE_PATHS['rgb']) # 加载rgb流的输入数据  tf.logging.info('RGB data loaded, shape=%s', str(rgb_sample.shape))  feed_dict[rgb_input] = rgb_sample  flow_sample = np.load(_SAMPLE_PATHS['flow']) # 加载flow流的输入数据  tf.logging.info('Flow data loaded, shape=%s', str(flow_sample.shape))  feed_dict[flow_input] = flow_sample  out_logits, out_predictions = sess.run(  [model_logits, model_predictions],  feed_dict=feed_dict)  out_logits = out_logits[0]  out_predictions = out_predictions[0]  sorted_indices = np.argsort(out_predictions)[::-1]  print('Inference time in sec: %.3f' % float(time.time() - start_time))  print('Norm of logits: %f' % np.linalg.norm(out_logits))  print('nTop classes and probabilities')  for index in sorted_indices[:20]:  print(out_predictions[index], out_logits[index], kinetics_classes[index]) WARNING:tensorflow:From /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version. Instructions for updating: Use standard file APIs to check for files with this prefix. INFO:tensorflow:Restoring parameters from data/checkpoints/rgb_imagenet/model.ckpt INFO:tensorflow:RGB checkpoint restored INFO:tensorflow:Restoring parameters from data/checkpoints/flow_imagenet/model.ckpt INFO:tensorflow:Flow checkpoint restored INFO:tensorflow:RGB data loaded, shape=(1, 79, 224, 224, 3) INFO:tensorflow:Flow data loaded, shape=(1, 79, 224, 224, 2) Inference time in sec: 1.511 Norm of logits: 138.468643 Top classes and probabilities 1.0 41.813675 playing cricket 1.497162e-09 21.49398 hurling (sport) 3.8431236e-10 20.13411 catching or throwing baseball 1.549242e-10 19.22559 catching or throwing softball 1.1360187e-10 18.915354 hitting baseball 8.801105e-11 18.660116 playing tennis 2.4415466e-11 17.37787 playing kickball 1.153184e-11 16.627766 playing squash or racquetball 6.1318893e-12 15.996157 shooting goal (soccer) 4.391727e-12 15.662376 hammer throw 2.2134352e-12 14.9772005 golf putting 1.6307096e-12 14.67167 throwing discus 1.5456218e-12 14.618079 javelin throw 7.6690325e-13 13.917259 pumping fist 5.1929587e-13 13.527372 shot put 4.2681337e-13 13.331245 celebrating 2.7205462e-13 12.880901 applauding 1.8357015e-13 12.487494 throwing ball 1.6134511e-13 12.358444 dodgeball 1.1388395e-13 12.010078 tap dancing