ft_audiovideobrowser.m

function ft_audiovideobrowser(cfg, data)

% FT_AUDIOVIDEOBROWSER reads and vizualizes the audio and/or video data
% corresponding to the EEG/MEG data that is passed into this function.
%
% Use as
%   ft_audiovideobrowser(cfg)
% or as
%   ft_audiovideobrowser(cfg, data)
% where the input data is the result from FT_PREPROCESSING or from FT_COMPONENTANALYSIS.
%
% The configuration structure can contain the following options
%   cfg.datahdr     = header structure of the EEG/MEG data, see FT_READ_HEADER
%   cfg.audiohdr    = header structure of the audio data, see FT_READ_HEADER
%   cfg.videohdr    = header structure of the video data, see FT_READ_HEADER
%   cfg.audiofile   = string with the filename
%   cfg.videofile   = string with the filename
%   cfg.trl         = Nx3 matrix, expressed in the MEG/EEG data samples, see FT_DEFINETRIAL
%   cfg.anonymize   = [x1 x2 y1 y2], range in pixels for placing a bar over the eyes (default = [])
%   cfg.interactive = 'yes' or 'no' (default = 'yes')
%
% If you do NOT specify cfg.datahdr, the header must be present in the input data.
% If you do NOT specify cfg.audiohdr, the header will be read from the audio file.
% If you do NOT specify cfg.videohdr, the header will be read from the video file.
% If you do NOT specify cfg.trl, the input data should contain a sampleinfo field.
%
% See also FT_DATABROWSER

% Copyright (C) 2015 Robert Oostenveld
%
% This file is part of FieldTrip, see http://www.fieldtriptoolbox.org
% for the documentation and details.
%
%    FieldTrip is free software: you can redistribute it and/or modify
%    it under the terms of the GNU General Public License as published by
%    the Free Software Foundation, either version 3 of the License, or
%    (at your option) any later version.
%
%    FieldTrip is distributed in the hope that it will be useful,
%    but WITHOUT ANY WARRANTY; without even the implied warranty of
%    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
%    GNU General Public License for more details.
%
%    You should have received a copy of the GNU General Public License
%    along with FieldTrip. If not, see <http://www.gnu.org/licenses/>.
%
% $Id$

persistent previous_audiofile previous_videofile previous_audiohdr previous_videohdr

% these are used by the ft_preamble/ft_postamble function and scripts
ft_revision = '$Id$';
ft_nargin   = nargin;
ft_nargout  = nargout;

% do the general setup of the function
ft_defaults
ft_preamble init
ft_preamble debug
ft_preamble loadvar data
ft_preamble provenance data

% the ft_abort variable is set to true or false in ft_preamble_init
if ft_abort
  return
end

% the data can be passed as input arguments or can be read from disk
hasdata = exist('data', 'var');

if hasdata
  data = ft_checkdata(data, 'datatype', {'raw+comp', 'raw'}, 'feedback', 'yes', 'hassampleinfo', 'yes');
end

% check if the input cfg is valid for this function
cfg = ft_checkconfig(cfg, 'renamed', {'anonimize', 'anonymize'}); % fix typo in previous version of the code
cfg = ft_checkconfig(cfg, 'renamed', {'anonymise', 'anonymize'}); % use North American and Oxford British spelling

% get the options from the user or set defaults
cfg.interactive = ft_getopt(cfg, 'interactive', 'yes');
cfg.anonymize   = ft_getopt(cfg, 'anonymize');
% the headers contain the information required for synchronization
cfg.datahdr     = ft_getopt(cfg, 'datahdr');
cfg.audiohdr    = ft_getopt(cfg, 'audiohdr');
cfg.videohdr    = ft_getopt(cfg, 'videohdr');
% the data is read on the fly
cfg.audiofile   = ft_getopt(cfg, 'audiofile');
cfg.videofile   = ft_getopt(cfg, 'videofile');

if isempty(cfg.videofile) && isempty(cfg.audiofile)
  ft_error('either cfg.videofile or cfg.audiofile should be specified');
end

if ~isempty(cfg.datahdr)
  % get it from the configuration
  datahdr = cfg.datahdr;
elseif hasdata
  % get it from the input data
  datahdr = ft_fetch_header(data);
else
  % in principle it would be possible to read it from cfg.datafile, but the
  % synchronization information will not automatically be present, as that
  % requires parsing one of the trigger channels
  ft_error('the data header is not available')
end

assert(isfield(datahdr, 'FirstTimeStamp'), 'sycnhronization information is missing in the data header');
assert(isfield(datahdr, 'TimeStampPerSample'), 'sycnhronization information is missing in the data header');

% determine the begin and end samples of the EEG/MEG data segments, the corresponding audio and video fragments will be displayed
if hasdata && isfield(data, 'sampleinfo')
  % construct the trial definition from the sampleinfo and the trialinfo
  trl = sampleinfo2trl(data);
elseif isfield(cfg, 'trl') && ischar(cfg.trl)
  % load the trial information from file
  trl = loadvar(cfg.trl, 'trl');
else
  % use the trial information that was specified
  trl = cfg.trl;
end

numtrl = size(trl,1);
trllop = 1;
while (true)
  
  fprintf('processing trial %d from %d\n', trllop, numtrl);
  audiodat = [];
  videodat = [];
  
  %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  if ~isempty(cfg.audiofile)
    if isequal(previous_audiofile, cfg.audiofile)
      audiohdr = previous_audiohdr;
    elseif ~isempty(cfg.audiohdr)
      fprintf('using the header and timestamps from the configuration\n');
      audiohdr = cfg.audiohdr;
    else
      fprintf('reading the header and timestamps from %s\n', cfg.audiofile);
      audiohdr = ft_read_header(cfg.audiofile);
    end
    
    % the FirstTimeStamp might be expressed as uint32 or uint64
    datahdr.FirstTimeStamp  = double(datahdr.FirstTimeStamp);
    audiohdr.FirstTimeStamp = double(audiohdr.FirstTimeStamp);
    
    begsample    = trl(trllop, 1); % expressed in the MEG/EEG data
    begtimestamp = (begsample-1)*datahdr.TimeStampPerSample + double(datahdr.FirstTimeStamp);
    begsample    = double(begtimestamp - audiohdr.FirstTimeStamp)/audiohdr.TimeStampPerSample + 1; % expressed in the audio data
    begsample    = round(begsample);
    
    endsample    = trl(trllop, 2); % expressed in the MEG/EEG data
    endtimestamp = cast((endsample-1)*datahdr.TimeStampPerSample, 'like', audiohdr.FirstTimeStamp) + datahdr.FirstTimeStamp;
    endsample    = double(endtimestamp - audiohdr.FirstTimeStamp)/audiohdr.TimeStampPerSample + 1; % expressed in the audio data
    endsample    = round(endsample);
    
    % deal with the case that the MEG/EEG and the audio recordings do not fully overlap
    begpad = 0;
    endpad = 0;
    if begsample<1
      begpad = 1-begsample;
      begsample = 1;
      ft_notice('padding the beginning of the audio with %d silent samples', begpad);
    end
    if endsample>audiohdr.nTrials*audiohdr.nSamples
      endpad = endsample-videohdr.nTrials*audiohdr.nSamples;
      endsample = audiohdr.nTrials*audiohdr.nSamples;
      ft_notice('padding the end of the audio with %d silent samples', endpad);
    end
    
    % read the audio data that corresponds to the selected MEG/EEG data
    ft_info('reading %d audio samples...', endsample-begsample+1);
    audiodat = ft_read_data(cfg.audiofile, 'begsample', begsample, 'endsample', endsample, 'header', audiohdr);
    ft_info('...done');
    
    if begpad || endpad
      audiodat = cat(2, zeros(audiohdr.nChans, begpad), audiodat, zeros(audiohdr.nChans, endpad));
    end
    
    % remember the header details to speed up subsequent calls
    previous_audiohdr  = audiohdr;
    previous_audiofile = cfg.audiofile;
  end
  
  %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  if ~isempty(cfg.videofile)
    if isequal(previous_videofile, cfg.videofile)
      videohdr = previous_videohdr;
    elseif ~isempty(cfg.videohdr)
      fprintf('using the header and timestamps from the configuration\n');
      videohdr = cfg.videohdr;
    else
      fprintf('reading the header and timestamps from %s\n', cfg.videofile);
      videohdr = ft_read_header(cfg.videofile);
    end
    
    % the FirstTimeStamp might be expressed as uint32 or uint64
    datahdr.FirstTimeStamp  = double(datahdr.FirstTimeStamp);
    videohdr.FirstTimeStamp = double(videohdr.FirstTimeStamp);
    
    begsample    = trl(trllop,1); % expressed in the MEG/EEG data
    begtimestamp = (begsample-1)*datahdr.TimeStampPerSample + double(datahdr.FirstTimeStamp);
    begsample    = double(begtimestamp - videohdr.FirstTimeStamp)/videohdr.TimeStampPerSample + 1; % expressed in the audio data
    begsample    = round(begsample);
    
    endsample    = trl(trllop,2); % expressed in the MEG/EEG data
    endtimestamp = cast((endsample-1)*datahdr.TimeStampPerSample, 'like', videohdr.FirstTimeStamp) + datahdr.FirstTimeStamp;
    endsample    = double(endtimestamp - videohdr.FirstTimeStamp)/videohdr.TimeStampPerSample + 1; % expressed in the audio data
    endsample    = round(endsample);
    
    % deal with the case that the MEG/EEG and the video recordings do not fully overlap
    begpad = 0;
    endpad = 0;
    if begsample<1
      begpad = 1-begsample;
      begsample = 1;
      ft_notice('padding the beginning of the video with %d empty frames', begpad);
    end
    if endsample>videohdr.nTrials*videohdr.nSamples
      endpad = endsample-videohdr.nTrials*videohdr.nSamples;
      endsample = videohdr.nTrials*videohdr.nSamples;
      ft_notice('padding the end of the video with %d empty frames', endpad);
    end
    
    % read the video data that corresponds to the selected MEG/EEG data, this takes some time
    ft_info('reading %d video frames...', endsample-begsample+1);
    videodat = ft_read_data(cfg.videofile, 'begsample', begsample, 'endsample', endsample, 'header', videohdr);
    ft_info('...done');
    
    if begpad || endpad
      videodat = cat(2, zeros(videohdr.nChans, begpad), videodat, zeros(videohdr.nChans, endpad));
    end
    
    if isfield(videohdr.orig, 'dim')
      % for VideoMEG files
      dim = [videohdr.orig.dim size(videodat,2)];
    else
      % for generic video files
      dim = [videohdr.orig.Height videohdr.orig.Width 3 size(videodat,2)];
    end
    
    videodat = uint8(videodat);
    videodat = reshape(videodat, dim);
    
    if ~isempty(cfg.anonymize)
      % place a bar over the eyes
      videodat(cfg.anonymize(1):cfg.anonymize(2), cfg.anonymize(3):cfg.anonymize(4), :) = 0;
    end
    
    % remember the header details to speed up subsequent calls
    previous_videohdr  = videohdr;
    previous_videofile = cfg.videofile;
  end
  
  %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  % start playing the video and/or audio
  
  if ~isempty(videodat)
    implay(videodat, videohdr.Fs);
    % FIXME this one does not play automatically
    drawnow
  end
  
  if ~isempty(audiodat)
    soundview(audiodat', audiohdr.Fs);
    % FIXME this one plays automatically
    drawnow
  end
  
  if isempty(videodat) && isempty(audiodat)
    ft_notice('there is no video and audio corresponding to the selected EEG/MEG data');
  end
  
  if istrue(cfg.interactive)
    response = 'x';
    while ~ismember(response, {'n', 'p', 'q'})
      response = input('press ''n'' for the next trial, ''p'' for the previous trial or ''q'' to quit: [N/p/q] ', 's');
    end
    switch response
      case 'n'
        if trllop==numtrl
          ft_warning('already at the last trial');
        else
          trllop = trllop+1;
        end
      case 'p'
        if trllop==1
          ft_warning('already at first trial');
        else
          trllop = trllop-1;
        end
      case 'q'
        break
    end
    
  else
    % not interactive, show the audio/video of all trials
    if trllop<numtrl
      trllop = trllop+1;
    else
      break
    end
    
  end % if interactive
  
end % while true

% do the general cleanup and bookkeeping at the end of the function
ft_postamble debug
ft_postamble previous data
ft_postamble provenance