using System; using System.Collections.Generic; using System.Text; using System.Windows.Forms; using TradeIdeas.Extensions; using TradeIdeas.MiscSupport; using TradeIdeas.ServerConnection; using TradeIdeas.TIProData; using TradeIdeas.TIProData.Interfaces; using TradeIdeas.TIProGUI; using TradeIdeas.XML; namespace TIProDevExtension { /// /// This is a special test rig I built to compare the recent changes to the chart server /// to the older chart server that's been in production for a while. /// /// I'm pointing to a micro_proxy with two special rules. candles_command_test maps to /// flex_command on my test server. candles_command_production maps to flex_command on /// the same group of servers that handle candles_command for production. I don't use /// candles_command directly because I don't know (or care) if that is pointing to test /// or production. /// /// I'm sending a list of requests to both servers and comparing the results. Ideally /// I'd hit a lot of special cases. Like a 15 minute candle with half of the data in /// the local cache and the other half in the database. But that's hard to do precisely, /// so I'm mostly relying on brute force. I'm asking for a lot of different variations. /// public partial class CompareChartData : Form { private IConnectionMaster _connectionMaster; private enum ServerType { Test, Production } private static String GetCommand(ServerType serverType) { if (serverType == ServerType.Test) return "candles_command_test"; else return "candles_command_production"; } private static String GetUserFriendly(ServerType serverType) { if (serverType == ServerType.Test) return "Test"; else return "Production"; } /// /// This is basically an interface required to use OneRequest. /// /// Originally OneRequest was only used by OneTest. They were closely intertwined. Since then /// OneTest has been renamed to OldNewTest. And We've added the new WeekMonthTest class which /// also uses OneRequest. Both OldNewTest and WeekMonthTest inherit from TestBase. /// private abstract class TestBase { public readonly CompareChartData Form; public TestBase(CompareChartData form) { Form = form; } public string Symbol { get; protected set; } private int _totalRetries = 0; private ulong _maxRetry = 0; public void ReportRetry(ulong epoch, ServerType serverType) { _totalRetries++; _maxRetry = Math.Max(_maxRetry, epoch); Form.retriesLabel.Text = "Total retries: " + _totalRetries + ", Max retry: " + _maxRetry + ", Last: " + GetUserFriendly(serverType); } public abstract void ReportResult(string data, ServerType serverType); protected void ShowText(string text) { Form.ShowText(text); } abstract public void ReportDebug(string message, ServerType serverType); abstract public void ReportFatalError(string message, ServerType serverType); } private class OneRequest { private readonly TestBase _owner; private readonly ServerType _serverType; private readonly string _prototype; private UInt64 _epoch = 0; private TalkWithServer.CancelToken _cancelToken; public OneRequest(TestBase owner, ServerType serverType, string prototype) { _owner = owner; _serverType = serverType; _prototype = prototype; SendRequestNow(); } private void CancelServerRequest() { _epoch++; if (null != _cancelToken) { _cancelToken.Cancel(); _cancelToken = null; } } private void SendRequestNow() { CancelServerRequest(); object[] message = new object[] { "command", GetCommand(_serverType), "subcommand", "make_and_show", "prototype", _prototype, "symbols", _owner.Symbol }; _cancelToken = new TalkWithServer.CancelToken(); _owner.Form._connectionMaster.SendManager.SendMessage (TalkWithServer.CreateMessage(message), OnResponse, streaming: true, clientId: _epoch, cancelToken: _cancelToken); } public void OnResponse(byte[] body, object clientId) { _owner.Form.BeginInvokeIfRequired(delegate () { UInt64 originalEpoch = (UInt64)clientId; if (originalEpoch != _epoch) // A response to a previous request. return; if (body == null) { _owner.ReportRetry(_epoch, _serverType); SendRequestNow(); } else { var document = XmlHelper.Get(body); if (document == null) ReportFatalError("Couldn't parse XML"); else { var top = document.DocumentElement; string type = top.Property("type"); if (type == "done") ReportFatalError("No data!"); else if (type == "debug") ReportDebug(top.Text()); else if (type == "error") ReportFatalError("Error message from server: " + top.Text()); else if (type == "result") { string symbol = top.Property("symbol"); if (symbol != _owner.Symbol) ReportFatalError("Wrong symbol. Expecting: " + _owner.Symbol + ". Found: " + symbol); else { ReportSuccess(top.Text()); } } else { // This is explicitly not an error. ReportDebug("Unexpected type: " + type); } } } }); } private void ReportSuccess(string data) { CancelServerRequest(); _owner.ReportResult(data, _serverType); } private void ReportDebug(string message) { _owner.ReportDebug(message, _serverType); } private void ReportFatalError(string message) { CancelServerRequest(); _owner.ReportFatalError(message, _serverType); } } static private char[] COMMA = new char[] { ',' }; static private char[] UNIX_END_OF_LINE = new char[] { '\n' }; /// /// Ask for the same data on the old and new servers and print any differences. /// private class OldNewTest : TestBase { override public void ReportFatalError(string message, ServerType serverType) { ShowText("Error from " + GetUserFriendly(serverType) + " " + _description + ": " + message); } override public void ReportDebug(string message, ServerType serverType) { ShowText("Debug from " + GetUserFriendly(serverType) + " " + _description + ": " + message); } class Row { static private bool CloseEnough(string field1, string field2) { if (field1 == field2) // Exact match. return true; double d1, d2; if (!(double.TryParse(field1, out d1) && double.TryParse(field2, out d2))) // At least one of these wasn't a number. We only allow exact match if it's not a number. return false; if ((d1 <= 0) || (d2 <= 0)) return false; if ((d2 == Math.Round(d2, 4)) && (d1 == Math.Round(d2, 4))) // The older server software would often round off to 4 digits after the decimal // when it shouldn't have. If that's the case, and we can round the newer one // to get the same answer, this is close enough. // Example: AAMMF, the close of the candle starting 2018-12-28 12:37:00 is 0.24119. // The older software would incorrectly round this to 0.2412. Many more // more examples of the same. return true; double CUTOFF = 0.000005; return Math.Abs(d2 - d1) / d1 < CUTOFF; } public bool CloseEnough(Row other) { string[] a1 = Original.Split(COMMA); string[] a2 = other.Original.Split(COMMA); if (a1.Length != a2.Length) return false; for (int i = 0; i < a1.Length; i++) if (!CloseEnough(a1[i], a2[i])) return false; return true; } public readonly string Key; public readonly string Original; public readonly string Source; public Row(string original, string source) { // We're using field 3 as the key. The first two fields are always the start and end time, // in a human readable format. For a real chart request the next two fields are the start // and end time in time_t format. We use the start time in time_t format because that should // be unique and it is easy to sort by. This includes some assumptions. If we change the // prototype we should keep the first column as the time or at least something unique. (The // first column of the request will be the thrid column of the response.) Key = original.Split(COMMA, 4)[2]; Original = original; Source = source; } public static IList MakeList(IList strings, string source) { List rows = new List(strings.Count); foreach (string s in strings) if (s != "") rows.Add(new Row(s, source)); return rows; } public static Dictionary MakeDictionary(IList rows) { Dictionary result = new Dictionary(); foreach (Row row in rows) result.Add(row.Key, row); return result; } public static void FindDifferences(IList source, Dictionary comparisons, IList differences) { foreach (Row row in source) { Row otherRow = null; comparisons.TryGetValue(row.Key, out otherRow); if ((otherRow == null) || !row.CloseEnough(otherRow)) differences.Add(row); } } public static int Compare(Row a, Row b) { int result = a.Key.CompareTo(b.Key); if (result != 0) return result; return a.Source.CompareTo(b.Source); } } private IList Compare(string s1, string s2) { IList list1 = Row.MakeList(s1.Split(UNIX_END_OF_LINE), "<<<<<< "); IList list2 = Row.MakeList(s2.Split(UNIX_END_OF_LINE), ">>>>>> "); var dictionary1 = Row.MakeDictionary(list1); var dictionary2 = Row.MakeDictionary(list2); List unmatchedRows = new List(); Row.FindDifferences(list1, dictionary2, unmatchedRows); Row.FindDifferences(list2, dictionary1, unmatchedRows); if (unmatchedRows.Count == 0) return null; unmatchedRows.Sort(Row.Compare); List result = new List(unmatchedRows.Count); foreach (Row row in unmatchedRows) result.Add(row.Source + row.Original); return result; } private readonly Dictionary _results = new Dictionary(); private string Summarize(ServerType serverType) { string responseFromServer; _results.TryGetValue(serverType, out responseFromServer); if (null == responseFromServer) return "No data (null)."; string[] rows = responseFromServer.Split(UNIX_END_OF_LINE); if (rows.Length == 2) // One header row on top, one blank row on the bottom. return "No data (header only)."; if (rows.Length < 3) return "No data (" + rows.Length + " rows)."; string startTime = rows[1].Split(COMMA)[0]; string endTime = rows[rows.Length - 2].Split(COMMA)[1]; return startTime + " - " + endTime + ", " + (rows.Length - 2) + " rows."; } override public void ReportResult(string data, ServerType serverType) { _results.Add(serverType, data); if (_results.Count == 2) { IList differences = Compare(_results[ServerType.Test], _results[ServerType.Production]); if (null == differences) ShowText("Success: " + _description + "; " + Summarize(ServerType.Test)); else { ShowText("Mismatch: " + _description); //ShowText("Symbol: " + _symbol); //ShowText("Prototype: " + _prototype); ShowText("<<<<<< Test >>>>>> Production"); foreach (string line in differences) ShowText(line); } } } private readonly String _description; private readonly OneRequest _testRequest; private readonly OneRequest _productionRequest; public OldNewTest(String prototype, String symbol, String description, CompareChartData form) : base(form) { Symbol = symbol; _description = description; _testRequest = new OneRequest(this, ServerType.Test, prototype); _productionRequest = new OneRequest(this, ServerType.Production, prototype); } } private class WeekMonthTest : TestBase { private string MakePrototype(int rowCount, string candleSize, long atTime) { StringBuilder prototype = new StringBuilder(); prototype.LAppend("daily"); prototype.LAppend(CHART_COLUMNS); StringBuilder options = new StringBuilder(); options.LAppend("row_count").LAppend(rowCount.ToString()); options.LAppend("pack").LAppend("1"); options.LAppend("candle_size").LAppend(candleSize); if (atTime > 0) options.LAppend("at_time").LAppend(atTime.ToString()); prototype.LAppend(options.ToString()); return prototype.ToString(); } public WeekMonthTest(CompareChartData form, string symbol) : base(form) { Symbol = symbol; bool weekly = _random.Next(2) == 0; // !weekly <--> monthly int rowCount = _random.Next(1, weekly ? 52 * 2 : 12 * 2); // 2 years. long atTime = 0; if (_random.Next(4) == 0) atTime = ServerFormats.ToTimeT(ServerFormats.Now.AddDays(_random.NextDouble() * -5)); string prototype = MakePrototype(rowCount, weekly ? "week" : "month", atTime); new OneRequest(this, ServerType.Test, prototype); } public override void ReportDebug(string message, ServerType serverType) { ShowText("Debug from WeekMonthTest(" + Symbol + "): " + message); } public override void ReportFatalError(string message, ServerType serverType) { System.Diagnostics.Debug.Assert(serverType == ServerType.Test); ReportError(message); } private void ReportError(string message) { ShowText("Error from WeekMonthTest(" + Symbol + "): " + message); } private class Row { public readonly long StartTime; public readonly long EndTime; public readonly double Open; public readonly double High; public readonly double Low; public readonly double Close; public readonly long Volume; Row(string asString) { string[] pieces = asString.Split(COMMA); StartTime = (long)double.Parse(pieces[2]); EndTime = (long)double.Parse(pieces[3]); Open = double.Parse(pieces[4]); High = double.Parse(pieces[5]); Low = double.Parse(pieces[6]); Close = double.Parse(pieces[7]); Volume = (long)double.Parse(pieces[8]); } public static List MakeAll(string csv) { List result = new List(); foreach (string row in csv.Split(UNIX_END_OF_LINE)) try { result.Add(new Row(row)); } catch { // Presumably this was the row at the top with all of the column // headings, or the row at the bottom which is empty. } return result; } public int CompareTo(Row other) { return StartTime.CompareTo(other.StartTime); } public string TimesString() { return ServerFormats.FromTimeT(StartTime).ToString() + " - " + ServerFormats.FromTimeT(EndTime).ToString(); } } private class RowAmalgamation { public readonly Row BigRow; public readonly List DailyRows = new List(); public RowAmalgamation(Row bigRow) { BigRow = bigRow; } public int CompareTo(RowAmalgamation other) { return BigRow.CompareTo(other.BigRow); } public class Comparer : IComparer { int IComparer.Compare(RowAmalgamation x, RowAmalgamation y) { return x.CompareTo(y); } } } private List _allRows; public override void ReportResult(string data, ServerType serverType) { System.Diagnostics.Debug.Assert(serverType == ServerType.Test); List rows = Row.MakeAll(data); if (null == _allRows) InitialResponse(rows); else DailyResponse(rows); } private void InitialResponse(List rows) { if (rows.Count == 0) { ShowText("Success from WeekMonthTest(" + Symbol + "): No data."); return; } _allRows = new List(rows.Count); foreach (Row row in rows) { _allRows.Add(new RowAmalgamation(row)); } _allRows.Sort((a, b) => a.CompareTo(b)); long atTime = _allRows[0].BigRow.StartTime; string prototype = MakePrototype(1, "day", atTime); new OneRequest(this, ServerType.Test, prototype); } private RowAmalgamation FindRowAmalgamation(Row row) { int index = _allRows.BinarySearch(new RowAmalgamation(row), new RowAmalgamation.Comparer()); if (index >= 0) return _allRows[index]; index = ~index; // index now points to the first row that starts after the time we are looking for. System.Diagnostics.Debug.Assert(index > 0); // We know that the time couldn't be before the first candle. DailyResponse checked for that // before calling FindRowAmalgamation. index--; // index now points to the last row that starts before the time we are looking for. RowAmalgamation possible = _allRows[index]; if (row.EndTime <= possible.BigRow.EndTime) return possible; // We can't find a good place for this row. Either it comes between two rows // in our list, it or comes after the last row in our list. return null; } private void DailyResponse(List rows) { long startTime = _allRows[0].BigRow.StartTime; int skipped = 0; foreach (Row row in rows) { if (row.StartTime < startTime) skipped++; else { RowAmalgamation addTo = FindRowAmalgamation(row); if (null == addTo) { ReportError("Can't find a place for daily candle " + row.TimesString() + "."); return; } addTo.DailyRows.Add(row); } } foreach (RowAmalgamation current in _allRows) { List dailyRows = current.DailyRows; if (dailyRows.Count == 0) { ReportError("No daily candles in range " + current.BigRow.TimesString() + "."); return; } dailyRows.Sort((a, b) => a.CompareTo(b)); double open = dailyRows[0].Open; double close = dailyRows[dailyRows.Count - 1].Close; double low = double.MaxValue; double high = double.MinValue; long volume = 0; foreach (Row row in dailyRows) { low = Math.Min(low, row.Low); high = Math.Max(high, row.High); volume += row.Volume; } if ((open != current.BigRow.Open) || (high != current.BigRow.High) || (low != current.BigRow.Low) || (close != current.BigRow.Close) || (volume != current.BigRow.Volume)) { ReportError("Mismatch " + current.BigRow.TimesString() + "Remote=(" + current.BigRow.Open + ", " + current.BigRow.High + ", " + current.BigRow.Low + ", " + current.BigRow.Close + ", " + current.BigRow.Volume + ") Local=(" + open + ", " + high + ", " + low + ", " + close + ", " + volume + ")."); return; } } ShowText("Success from WeekMonthTest(" + Symbol + "). Oldest big candle " + _allRows[0].BigRow.TimesString() + ". " + skipped + " daily row(s) before that."); } } public CompareChartData(IConnectionMaster connectionMaster) { _connectionMaster = connectionMaster; InitializeComponent(); } private static readonly Random _random = new Random(); private static readonly string CHART_COLUMNS = "{START {{current(\"start\")}}} {END {{current(\"end\")}}} {OPEN {{}}} {HIGH {{}}} {LOW {{}}} {CLOSE {{}}} {VOLUME {{}}} {SWING_EXIT {{}}} {AVG_VOLUME {{average(30, 1, \"VOLUME\", 1)}}}"; private static string RandomPrototype(StringBuilder description) { int rowCount = 0; switch (_random.Next(3)) { case 0: rowCount = 350; break; case 1: rowCount = 60; break; case 2: rowCount = _random.Next(1, 350); break; } System.Diagnostics.Debug.Assert(rowCount > 0); description.Append(" row_count="); description.Append(rowCount); bool intraDay = _random.Next(2) > 0; StringBuilder result = new StringBuilder(); // TODO We should really be using LAppend() instead of Append(). See the examples below. if (intraDay) result.Append("intraday {"); else result.Append("daily {"); result.Append(CHART_COLUMNS); result.Append("} {row_count "); result.Append(rowCount); result.Append(" pack 1"); if (intraDay) { int[] options = new int[] { 1, 2, 3, 5, 10, 15, 60 }; result.Append(" minutes_per_candle "); int minutesPerCandle = options[_random.Next(0, options.Length)]; result.Append(minutesPerCandle); description.Append(" minutes_per_candle="); description.Append(minutesPerCandle); if (_random.Next(2) == 0) { result.Append(" start_offset 60 end_offset 60"); description.Append(" include_pre_and_post"); } } else description.Append(" daily"); if (_random.Next(5) == 0) { DateTime time = ServerFormats.Now; if (intraDay && (_random.Next(3) != 0)) time = time.AddDays(_random.NextDouble() * -3); else time = time.AddDays(_random.NextDouble() * -700); result.Append(" at_time "); result.Append(ServerFormats.ToTimeT(time).ToString()); description.Append(" at_time="); description.Append(time); } result.Append('}'); return result.ToString(); } // These initial symbols were chosen because they've each had a split recently. private string[] _symbols = new string[] { "CATB", "IRET", "CEI", "SRC", "ROL", "AINV", "CASS", "GNBT", "LARK", "NULTF", "CBSH", "BMRC", "ITUB", "TNXP", "TRI" }; private int _symbolIndex = -1; private string NextSymbol() { _symbolIndex++; if (_symbolIndex >= _symbols.Length) _symbolIndex = 0; return _symbols[_symbolIndex]; } private void ShowText(string text) { textBox1.AppendText(text); textBox1.AppendText("\r\n"); } private static HashSet _symbolBlackList = new HashSet { }; private IList NextSymbols() { List result = new List(); int count; if ((!int.TryParse(countTextBox.Text, out count)) || (count < 1)) { ShowText("\"" + countTextBox.Text + "\" is not a valid count"); } else { result.Capacity = count; for (int i = 0; i < count; i++) { string symbol = NextSymbol(); if (_symbolBlackList.Contains(symbol)) // It's tempting to try grabbing another symbol. But that might lead to an // infinite loop. ShowText("Skipping: " + symbol); else result.Add(symbol); } } return result; } private void oldNewTestButton_Click(object sender, EventArgs e) { foreach (string symbol in NextSymbols()) { StringBuilder description = new StringBuilder("Random Prototype"); String prototype = RandomPrototype(description); description.Append(" Symbol="); description.Append(symbol); new OldNewTest(prototype, symbol, description.ToString(), this); } } public void SymbolListResponse(byte[] body, object clientId) { this.BeginInvokeIfRequired(delegate () { if (null == body) RequestSymbolList(); else { _symbols = TalkWithServer.BytesToStr(body).Split(new char[] { ' ' }); ShowText("Loaded " + _symbols.Length + " symbols from server."); if (_symbols.Length == 0) _symbols = new string[] { "A", "T", "INTC" }; } }); } private void RequestSymbolList() { object[] message = new object[] { "command", GetCommand(ServerType.Test), "subcommand", "what_symbols_are_we_following" }; _connectionMaster.SendManager.SendMessage(TalkWithServer.CreateMessage(message), SymbolListResponse); } private void getSymbolListButton_Click(object sender, EventArgs e) { RequestSymbolList(); } private void restartSymbolsButton_Click(object sender, EventArgs e) { _symbolIndex = -1; } private void weekMonthTestButton_Click(object sender, EventArgs e) { foreach (string symbol in NextSymbols()) { new WeekMonthTest(this, symbol); } } } /// /// The standard way to add an extension to the menu. /// class CompareChartDataExtension : IMainInit { /// /// Normal init for extensions. /// /// /// public static IList GetExtensions(Object mainProgram) { List result = new List(1); result.Add(new CompareChartDataExtension((IMainProgram)mainProgram)); return result; } private readonly IMainProgram _mainProgram; private CompareChartDataExtension(IMainProgram mainProgram) { _mainProgram = mainProgram; } public void MainInit() { // ♥ = U+2665 = Black Heart Suit _mainProgram.AddToToolsMenu("Compare Chart Data ♥", delegate { new CompareChartData(_mainProgram.ConnectionMaster).Show(); }); } } }