faster rcnn test demo ---repaired for video input and save the image, label, score et al. into .mat format

faster rcnn test demo ---repaired for video input and save the image, label, score et al. into .mat format



function script_faster_rcnn_demo()
close all;
clc;
clear mex;
clear is_valid_handle; % to clear init_key
run(fullfile(fileparts(fileparts(mfilename('fullpath'))), 'startup'));
%% -------------------- CONFIG --------------------
opts.caffe_version          = 'caffe_faster_rcnn';
opts.gpu_id                 = auto_select_gpu;
active_caffe_mex(opts.gpu_id, opts.caffe_version);

opts.per_nms_topN           = 6000;
opts.nms_overlap_thres      = 0.7;
opts.after_nms_topN         = 300;
opts.use_gpu                = true;

opts.test_scales            = 600;

% %% -------------------- INIT_MODEL --------------------
% model_dir                   = fullfile(pwd, 'output', 'faster_rcnn_final', 'faster_rcnn_VOC0712_vgg_16layers'); %% VGG-16
model_dir                   = fullfile(pwd, 'output', 'faster_rcnn_final', 'faster_rcnn_VOC0712_ZF'); %% ZF
proposal_detection_model    = load_proposal_detection_model(model_dir);

proposal_detection_model.conf_proposal.test_scales = opts.test_scales;
proposal_detection_model.conf_detection.test_scales = opts.test_scales;
if opts.use_gpu
    proposal_detection_model.conf_proposal.image_means = gpuArray(proposal_detection_model.conf_proposal.image_means);
    proposal_detection_model.conf_detection.image_means = gpuArray(proposal_detection_model.conf_detection.image_means);
end

% caffe.init_log(fullfile(pwd, 'caffe_log'));
% proposal net
rpn_net = caffe.Net(proposal_detection_model.proposal_net_def, 'test');
rpn_net.copy_from(proposal_detection_model.proposal_net);
% fast rcnn net
fast_rcnn_net = caffe.Net(proposal_detection_model.detection_net_def, 'test');
fast_rcnn_net.copy_from(proposal_detection_model.detection_net);

% set gpu/cpu
if opts.use_gpu
    caffe.set_mode_gpu();
else
    caffe.set_mode_cpu();
end       

%% -------------------- WARM UP --------------------
% the first run will be slower; use an empty image to warm up

for j = 1:2     % we warm up 2 times
    im = uint8(ones(375, 500, 3)*128);
    if opts.use_gpu
        im = gpuArray(im);
    end
    [boxes, scores]        = proposal_im_detect(proposal_detection_model.conf_proposal, rpn_net, im);
    aboxes                      = boxes_filter([boxes, scores], opts.per_nms_topN, opts.nms_overlap_thres, opts.after_nms_topN, opts.use_gpu);
    if proposal_detection_model.is_share_feature
        [boxes, scores]             = fast_rcnn_conv_feat_detect(proposal_detection_model.conf_detection, fast_rcnn_net, im, ...
            rpn_net.blobs(proposal_detection_model.last_shared_output_blob_name), ...
            aboxes(:, 1:4), opts.after_nms_topN);    
    else
        [boxes, scores]             = fast_rcnn_im_detect(proposal_detection_model.conf_detection, fast_rcnn_net, im, ...
            aboxes(:, 1:4), opts.after_nms_topN);
    end
end

%% -------------------- TESTING --------------------
% im_names = {'001763.jpg', '004545.jpg', '000542.jpg', '000456.jpg', '001150.jpg'};
% these images can be downloaded with fetch_faster_rcnn_final_model.m
% for j = 1:length(im_names)
% im = imread(fullfile(pwd, im_names{j}));


%% -------------------------- Video ------------------
%% initialization 
video_names = {};


%% --------------path for test ------------
path = '/media/wangxiao/247317a3-e6b5-45d4-81d1-956930526746/video/test_video/'; 
savePath = '/media/wangxiao/247317a3-e6b5-45d4-81d1-956930526746/video/test_video_images/';


%% ---------------path for the 4TB drive----------
% path = '/media/wangxiao/Seagate/pedestrian data/';
% savePath = '/media/wangxiao/Seagate/pedestrian data/pedestrian frame/';

% video_dir = dir(path);

    video_dir = dir(fullfile(path, '*.avi'));
%    if isempty(video_dir)
%        video_dir = dir(fullfile(path, '*.mp4'));
%    end
%    if isempty(video_dir)
%       video_dir = dir(fullfile(path, '*.rmvb'));
%    end
%    if isempty(video_dir)
%       video_dir = dir(fullfile(path, '*.mkv'));
%    end


%% 
k =1;
for i = 1:length(video_dir)
    if ~video_dir(i).isdir
        video_names{k} = video_dir(i).name;       % 视频的名字　保存在video_name中；
        k = k+1;
    end
end
clear k;
running_time = [];

%% 将视频load进来，切割为图像，存储 
for j = 1:length(video_dir)  %视频级别；
         
            % 弹出错误提示： Could not read file due to an unexpected error. Reason:
            % Unable to initialize the video obtain properties ...
            frames = VideoReader (strcat(path, video_dir(j).name));
            numFrames =frames.NumberOfFrames;
        for k = 1 : 30: numFrames
            disp(['saving the ',num2str(k),'-th frames, please waiting....']);
            frame = read(frames,k);   %将frame存储起来；
            frame =  imresize(frame, [200, 300]);       % 是否要对图像进行resize ?
			imwrite(frame,[savePath,sprintf('%08d.png',k)]);
        end
end


%% 读取图像，输入给　faster rcnn
image_path = savePath;
imageFile = dir([image_path, '*.png']);

for iii = 1:length(imageFile)
    im = imread([image_path, imageFile(iii).name]);
        
    if opts.use_gpu
        im = gpuArray(im);
    end
    
  
    
    % test proposal
    th = tic();
    [boxes, scores]             = proposal_im_detect(proposal_detection_model.conf_proposal, rpn_net, im);
    t_proposal = toc(th);
    th = tic();
    aboxes                           = boxes_filter([boxes, scores], opts.per_nms_topN, opts.nms_overlap_thres, opts.after_nms_topN, opts.use_gpu);
    t_nms = toc(th);
    
    % test detection
    th = tic();
    if proposal_detection_model.is_share_feature
            [boxes, scores]             = fast_rcnn_conv_feat_detect(proposal_detection_model.conf_detection, fast_rcnn_net, im, ...
            rpn_net.blobs(proposal_detection_model.last_shared_output_blob_name), ...
            aboxes(:, 1:4), opts.after_nms_topN);
        
    else
            [boxes, scores]             = fast_rcnn_im_detect(proposal_detection_model.conf_detection, fast_rcnn_net, im, ...
            aboxes(:, 1:4), opts.after_nms_topN);
        
    end
    t_detection = toc(th);
    
    fprintf('%s (%dx%d): time %.3fs (resize+conv+proposal: %.3fs, nms+regionwise: %.3fs)\n', ['the ' num2str(j) '-th image '], size(im, 2), size(im, 1), t_proposal + t_nms + t_detection, t_proposal, t_nms+t_detection);
    running_time(end+1) = t_proposal + t_nms + t_detection;
    
   %%  visualize 可视化；
    classes = proposal_detection_model.classes;  % 20类物体，包括行人这一类；
    boxes_cell = cell(length(classes), 1);   % 20*1 cell;
    thres = 0.6;
    
    for i = 1:length(boxes_cell)
        boxes_cell{i} = [boxes(:, (1+(i-1)*4):(i*4)), scores(:, i)];
        boxes_cell{i} = boxes_cell{i}(nms(boxes_cell{i}, 0.3), :);
        
        I = boxes_cell{i}(:, 5) >= thres;
        boxes_cell{i} = boxes_cell{i}(I, :);
    end
    
        figure(j);
        
        [location, label, score] = output(im, boxes_cell, classes, 'voc');           
        if (score==0)
             continue;
        else
%              disp(imageFile(iii).name);
             save(['./mat results/' imageFile(iii).name '.mat' ], 'location', 'label', 'score', 'im');
        end
  
        showboxes(im, boxes_cell, classes, 'voc');

        pause(0.1);

end
 
%      eval(['movie_' num2str(k) '_' class_filenames{NUM} '= im_names;']);
%      clear im_names;
%      
%      eval(['movie_bb_' num2str(k) '_' class_filenames{NUM} '= video_cell;'])
%      clear video_cell;
%     
%     if ~exist([fullfile(pwd, 'video_frames'),'/','video.mat'],'file')
%         save('./video_frames/video.mat',['movie_' num2str(k) '_' class_filenames{NUM}]);
%         save('./video_frames/video_bb.mat',['movie_bb_' num2str(k) '_' class_filenames{NUM}]);
%     else
%         save('./video_frames/video.mat',['movie_' num2str(k) '_' class_filenames{NUM}],'-append');
%         save('./video_frames/video_bb.mat',['movie_bb_' num2str(k) '_' class_filenames{NUM}], '-append');
%     end
%     
%     eval(['clear movie_' num2str(k)]);
%     eval(['clear movie_bb_' num2str(k)]);
%     save('im_boxes.mat','im_cell');

    fprintf('mean time: %.3fs\n', mean(running_time));
    caffe.reset_all(); 
    clear mex;

end



function proposal_detection_model = load_proposal_detection_model(model_dir)
    ld                          = load(fullfile(model_dir, 'model'));
    proposal_detection_model    = ld.proposal_detection_model;
    clear ld;
    
    proposal_detection_model.proposal_net_def ...
                                = fullfile(model_dir, proposal_detection_model.proposal_net_def);
    proposal_detection_model.proposal_net ...
                                = fullfile(model_dir, proposal_detection_model.proposal_net);
    proposal_detection_model.detection_net_def ...
                                = fullfile(model_dir, proposal_detection_model.detection_net_def);
    proposal_detection_model.detection_net ...
                                = fullfile(model_dir, proposal_detection_model.detection_net);
    
end

function aboxes = boxes_filter(aboxes, per_nms_topN, nms_overlap_thres, after_nms_topN, use_gpu)
    % to speed up nms
    if per_nms_topN > 0
        aboxes = aboxes(1:min(length(aboxes), per_nms_topN), :);
    end
    % do nms
    if nms_overlap_thres > 0 && nms_overlap_thres < 1
        aboxes = aboxes(nms(aboxes, nms_overlap_thres, use_gpu), :);       
    end
    if after_nms_topN > 0
        aboxes = aboxes(1:min(length(aboxes), after_nms_topN), :);
    end
end