//+------------------------------------------------------------------+
//|                                        Plot COT futures data.mq4 |
//+------------------------------------------------------------------+

#property  indicator_separate_window

#include <hanover --- function header.mqh>

#property  indicator_buffers 1
#property  indicator_color1  Red
#property  indicator_width1  1

extern string     JoinFiles        = "";
extern int        COTItemType      = 1;
extern int        COTColumnNo      = 8;

string   RefreshPeriod    = "D1";
int      DateColumnNo     = 3;

string   DateFormat       = "YYYY.MM.DD HH:II";
string   DateDelimiters   = ". :";
string   FieldDelimiter   = ",";

datetime   prev_time, dt[9999];
double     val[9999], buffer0[];
string     FileName, item[999], value[999], aa[200], file[10], sym, arr[4];
bool       FirstTime;
int        RefreshEveryXMins;

//+------------------------------------------------------------------+
int init()  {
//+------------------------------------------------------------------+
  SetIndexBuffer(0,buffer0);
  SetIndexStyle(0,DRAW_LINE,STYLE_SOLID);
  SetIndexDrawBegin(0,0);
  IndicatorShortName(NumberToStr(COTItemType,"'COT['T5','")+NumberToStr(COTColumnNo,"T5']'"));

  sym               = Symbol();
  RefreshEveryXMins = StrToTF(RefreshPeriod);
  prev_time         = -9999;

  FileName = NumberToStr(GetUniqueInt(),"'COT_Futures'T5'.CSV'");

  if (JoinFiles() >= 0)
    BuildOutputFile();
  
  return(0);
}

//+------------------------------------------------------------------+
int JoinFiles()  {
//+------------------------------------------------------------------+
  if (StringLen(JoinFiles) < 1)  return(0);                  // no join required, simply process annual.txt file
  int NumFiles = StrToStringArray(JoinFiles,file,",");
  string fname = "annual.txt";
  int ho = FileOpen(fname, FILE_CSV|FILE_WRITE,'~');
  for (int i=0; i<NumFiles; i++)   {                         // loop for each input file
    int hi = FileOpen(file[i], FILE_CSV|FILE_READ,'~');      // open the input file
    if (hi<=0) {
      Comment("File '"+file[i]+"' not found in ....\experts\files folder.");
      FileClose(ho);
      return(-1);                                            // error - couldn't find one of the files to be joined
    }
    while (!FileIsEnding(hi))   {                            // process the input file
      string s = FileReadString(hi);
      FileWrite(ho,s);                                       // write data to output file
    }
    FileClose(hi);                                           // close input file
  }  
  FileClose(ho);                                             // close output file
  return(1);                                                 // return without error
}

//+------------------------------------------------------------------+
int BuildOutputFile()  {
//+------------------------------------------------------------------+
// Load data from "COT futures item types.csv" file into item array......
  string fname = "COT futures item types.csv";
  int h = FileOpen(fname, FILE_CSV|FILE_READ,'~');
  if (h<=0) {
    Comment("File '"+fname+"' not found in ....\experts\files folder.");
    return(0);
  }
  int z=0;  
  while (!FileIsEnding(h))   {
    z++;
    item[z] = FileReadString(h);
  }
  FileClose(h);
//+------------------------------------------------------------------+
// Load data from "COT futures values.csv" file into value array......
  fname = "COT futures values.csv";
  h = FileOpen(fname, FILE_CSV|FILE_READ,'~');
  if (h<=0) {
    Comment("File '"+fname+"' not found in ....\experts\files folder.");
    return(0);
  }
  z=7;  
  while (!FileIsEnding(h))   {
    z++;
    value[z] = FileReadString(h);
  }
  FileClose(h);
//+------------------------------------------------------------------+
// Build the file "COT_Futures<n>.CSV" from the data in the annual1.txt file.....
  int ho = FileOpen(FileName, FILE_CSV|FILE_WRITE,'~');
  fname = "annual.txt";
  int hi = FileOpen(fname, FILE_CSV|FILE_READ,'~');
  if (hi<=0) {
    Comment("File '"+fname+"' not found in ....\experts\files folder.");
    return(0);
  }
  while (!FileIsEnding(hi))   {
    string s = FileReadString(hi);
    z = StringFind(s,"\x22",1);
    if (StringSubstr(s,1,z-1) != item[COTItemType])     continue;              // skip if not user-specified item type
    StrToStringArray(StringSubstr(s,z+2),aa,",");                              // parse comma-separated data string into aa array
    double    val = StrToNumber(aa[COTColumnNo-1]);                            // extract the value from the user-specified column#
    datetime  dt  = StrToDate(aa[DateColumnNo-1],"YYYY-MM-DD");                // extract the date from column #3
    FileWrite(ho, DateToStr(dt,"Y.M.D H:I")+NumberToStr(val,"','T9.2"));       // output the date and value to the output file
//    log(item[COTItemType],aa[COTColumnNo-1],aa[DateColumnNo-1]);
  }
  FileClose(hi);
  FileClose(ho);
  FileSort(FileName);                                                          // sort the output file's records into ascending date sequence
  return(0);
}

//+------------------------------------------------------------------+
int start()  {
//+------------------------------------------------------------------+
  if (RefreshEveryXMins < 0)  {
    if (FirstTime)  {
      plot_data();
    }
    FirstTime = false;      
    return(0);
  }  
  if (RefreshEveryXMins == 0) {
    plot_data();    
  } else {
    if (prev_time != iTime(sym,RefreshEveryXMins,0))  {
      plot_data();
      prev_time = iTime(sym,RefreshEveryXMins,0);
  } }      
  return(0);
}

//+------------------------------------------------------------------+
int plot_data()  {
//+------------------------------------------------------------------+
// plot_data() is based on Plot_External_Data.mq4 logic
// Read the "COT_Futures<n>.CSV" file and load the data into the buffer that gets plotted.......
  int h = FileOpen(FileName, FILE_CSV|FILE_READ,'~');
  if (h<=0) {
    Comment("File "+FileName+" not found.");
    return(0);
  }  

  // First pass loads data from file into arrays dt, val   (i.e. date/time, and value).....
  for (int c=0; !FileIsEnding(h) && c<9999; c++)  {
    string tmp = FileReadString(h);
    if (FileIsEnding(h))  break;
    StrToStringArray(tmp,arr,FieldDelimiter);
    dt[c]  = StrToDate(arr[0],DateFormat,DateDelimiters);
    val[c] = StrToNumber(arr[1]);
//    if (c>0)  log(c,DateToStr(dt[c]),NumberToStr(val[c],"RT-3.5"));    // debugging only
  }
  FileClose(h);
  
  // Second pass re-synchs data from dt, val arrays onto chart timeframe, and plots them accordingly.....
  c--;
  for (int i=0; i<=Bars; i++)  {
    while(Time[i] < dt[c]) c--;
    if (c >= 0) buffer0[i] = val[c];
//    if (i<30) log(i,DateToStr(Time[i]),c,DateToStr(dt[c]),val[c],buffer0[i]);    // debugging only
  }
  return(0);
}

//+------------------------------------------------------------------+
#include <hanover --- extensible functions.mqh>