Download All Tick Data for Symbols in MetaTrader 5

If you're looking to download all available tick data for a specific symbol in MetaTrader 5, this guide is for you! This nifty expert advisor will scan your broker's market watch and pull all the ticks or ticks up to a specified date.

Having access to comprehensive historical data is crucial for backtesting your strategies or creating custom charts based on ticks. Just a heads-up: be sure you have enough hard drive space, as the terminal stores this data in the data folder.

Before we dive into the code, we'll need a download manager to facilitate the download process. The CDownloadManager structure will hold everything we need.

struct CDownloadManager
  {
   bool m_started, m_finished;
   string m_symbols[], m_current;
   int m_index;
}

State of the download (started/finished)
List of symbols to scan
Current symbol being processed
Index of the symbol being scanned

We’ll also need functions to read and write to the hard drive, especially since we’re working with symbols. Here’s how we can write a string to a file:

void writeStringToFile(int f, string thestring)
  {
    // Save symbol string
    char sysave[];
    int charstotal = StringToCharArray(thestring, sysave, 0, StringLen(thestring), CP_ACP);
    FileWriteInteger(f, charstotal, INT_VALUE);
    for(int i = 0; i < charstotal; i++)
      {
        FileWriteInteger(f, sysave[i], CHAR_VALUE);
      }
}

This function takes:

File handle f (opened for writing with binary flags FILE_WRITE|FILE_BIN)
The string to write to the file

It writes the length of the string followed by each character. To load the string from a file, here’s how you can do it:

string readStringFromFile(int f)
  {
    string result = "";
    // Load symbol string
    char syload[];
    int charstotal = (int)FileReadInteger(f, INT_VALUE);
    if(charstotal > 0)
    {
      ArrayResize(syload, charstotal, 0);
      for(int i = 0; i < charstotal; i++)
        {
        syload[i] = (char)FileReadInteger(f, CHAR_VALUE);
      }
      result = CharArrayToString(syload, 0, charstotal, CP_ACP);
    }
    return(result);
}

This function takes:

File handle f (opened for reading as binary, flags FILE_READ|FILE_BIN)

It reads the integer length of characters expected, processes them into a char array, and returns the result as a string.

Now, let's initialize the download manager and fill it with symbols from the market watch:

//+------------------------------------------------------------------+
//| Grab symbols from the market watch |
//+------------------------------------------------------------------+
void grab_symbols()
  {
    //! Only from the market watch!
    int s = SymbolsTotal(true);
    ArrayResize(m_symbols, s, 0);
    for(int i = 0; i < ArraySize(m_symbols); i++)
      {
        m_symbols[i] = SymbolName(i, true);
      }
  }

Here's a quick rundown of what this function does:

It checks how many symbols are in the market watch (active)
Resizes the m_symbols array to accommodate them
Loops through the total symbols to request their names

Next, we need a function to manage the download process:

//+------------------------------------------------------------------+
//| Manage the download of symbols process |
//+------------------------------------------------------------------+
void manage(string folder, string filename)
  {
    // Essentially this starts or navigates to the next symbol
    // If set
    if(ArraySize(m_symbols) > 0)
    {
      // If not started
      if(!m_started)
        {
        m_started = true;
        // Go to first symbol
        m_current = m_symbols[0];
        m_index = 1;
        save(folder, filename);
        if(_Symbol != m_current)
        {
          ChartSetSymbolPeriod(ChartID(), m_current, _Period);
        }
        else
        {
          ENUM_TIMEFRAMES new_period = PERIOD_M1;
          for(int p = 0; p < ArraySize(TFS); p++)
            {
             if(_Period != TFS[p])
            {
            new_period = TFS[p];
             break;
            }
          }
        ChartSetSymbolPeriod(ChartID(), m_current, new_period);
      }
      return;
    }
    // If started
    else
    {
      m_index++;
      if(m_index <= ArraySize(m_symbols))
      {
        m_current = m_symbols[m_index - 1];
        save(folder, filename);
        if(_Symbol != m_current)
        {
          ChartSetSymbolPeriod(ChartID(), m_current, _Period);
        }
      return;
      }
      else
      {
        m_finished = true;
        FileDelete(folder + "\" + filename);
        Print("Finished");
        ExpertRemove();
        return;
      }
    }
    else
    {
      Print("Please grab symbols first");
    }
    // End of set
  }

Here’s a quick summary of how it works:

Open a chart (you only need one) and set a timer.
The timer triggers the download process.
Check whether this is a new or continuing download.
If it’s new, grab all the symbols.
If it’s ongoing, download data for the current symbol.

This is the code handling downloads on a timer:

//+------------------------------------------------------------------+
//| Timer |
//+------------------------------------------------------------------+
void OnTimer()
  {
    //--- If synchronized
    if(SymbolIsSynchronized(_Symbol) && TerminalInfoInteger(TERMINAL_CONNECTED) == 1)
    {
      EventKillTimer();
      //--- Load the system here
      if(MANAGER.load(MANAGER_FOLDER, MANAGER_STATUS_FILE))
        {
        //--- System loaded, processing symbol
        Comment("System loaded and we are processing " + MANAGER.m_current);
        //--- Tick load
        //--- Find the oldest tick available in the broker
        int attempts = 0;
        int ping = -1;
        datetime cursor = flatten(TimeTradeServer());
        long cursorMSC = ((long)cursor) * 1000;
        long jump = 2592000000;
        MqlTick receiver[];
        long oldest = LONG_MAX;
        Comment("Please wait");
        while(attempts < 5)
          {
            ping = CopyTicks(_Symbol, receiver, COPY_TICKS_ALL, cursorMSC, 1);
            if(ping == 1)
              {
                if(receiver[0].time_msc == oldest)
                {
                  attempts++;
                }
                else
                {
                attempts = 0;
                }
                if(receiver[0].time_msc < oldest)
                {
                  oldest = receiver[0].time_msc;
                }
              cursorMSC -= jump;
              if(limitDate && receiver[0].time <= oldestLimit)
                {
                  break;
                }
          }
        else
          {
            attempts++;
          }

            Sleep(44);
            Comment("Oldest Tick : " + TimeToString((datetime)(oldest / 1000), TIME_DATE | TIME_MINUTES | TIME_SECONDS) + "\nCursor(" + TimeToString((datetime)(cursorMSC / 1000), TIME_DATE | TIME_MINUTES | TIME_SECONDS) + ")\nAttempts(" + IntegerToString(attempts) + ")\nPlease wait for response...");
        }
        //--- At this point we have the oldest tick
        //--- Start requesting ticks from the oldest to the newest
        if(oldest != LONG_MAX)
        {
            ArrayFree(receiver);
            datetime newest_tick = 0;
            //--- Receive the time of the last tick for this symbol
            datetime most_recent_candle = (datetime)SymbolInfoInteger(_Symbol, SYMBOL_TIME);
            while(newest_tick < most_recent_candle)
            {
               //--- Request a new batch starting from the oldest time
              int pulled = CopyTicks(_Symbol, receiver, COPY_TICKS_ALL, oldest, tick_packets);
              if(pulled > 0)
                {
                  //--- If we pull a new batch, update our downloaded times
                  newest_tick = receiver[pulled - 1].time;
                  oldest = receiver[pulled - 1].time_msc;
                  ArrayFree(receiver);
                }
              //--- Timeout server requests, alter if needed
              Sleep(44);
              Comment("Pulled up to " + TimeToString(newest_tick, TIME_DATE | TIME_MINUTES | TIME_SECONDS) + " so far");
          }
        }
        else
        {
            Alert("Please close the terminal \n head over to the ticks folder \n and delete the empty folders");
            ExpertRemove();
      }
        //--- Update the manager and move on
        MANAGER.manage(MANAGER_FOLDER, MANAGER_STATUS_FILE);
      }
      else
      {
        //--- Grab the market watch symbols to start download
        Comment("Grabbing MW and starting");
        MANAGER.grab_symbols();
        MANAGER.manage(MANAGER_FOLDER, MANAGER_STATUS_FILE);
      }
  }

And there you have it! With this code, you're all set to download tick data for your symbols in MetaTrader 5. Happy trading!

Download All Tick Data for Symbols in MetaTrader 5

Related Posts