Unit ProcessData;

Interface

  Uses
    TwoDLookup, TwoDArrayWriters, DataFormats, DumpCandles, MultiLog;

  Type
    TProcessData =
      Class(TObject)
      Public
        { These set the comparison points that come from other stocks. }
        Procedure AddPossibleCorrelation(Symbol : String; Data1Day : TBarList);
        Procedure AddFuturesCorrelation(Data1Day : TBarList);

        Procedure ProcessData(Symbol : String;
                              Data1Min, Data1Day : TBarList;
                              DataHighs, DataLows : TPriceList;
                              DataFundamental : TFundamentalData;
                              CorporateActions : TCorporateActions);
        Procedure SetOutputFiles(BaseName : String);
        Procedure FlushOutputFiles;
        Procedure ClearAll;
        Procedure SetPeriods(P : Integer);
        Procedure SetStartTime(T : Double);
        Procedure InitializeDumpCandles;
        Constructor Create;
        Destructor Destroy; Override;
      Private
        OutputData : TTwoDArrayWriter;
        VolumeData : TTwoDArrayWriter;
        TCData : TTwoDArrayWriter;
        VolumeBlocks : TTwoDArrayWriter;
        StandardCandles : TTwoDArrayWriter;
        FundamentalData : TTwoDArrayWriter;
        PristineData : TTwoDArrayWriter;
        AdditionalData : TTwoDArrayWriter;
        PossibleCorrelations : Array Of
          Record
            Symbol : String;
            Bars : TBarList;
          End;
        FuturesDailyData : TBarList;
        FChartsDirectory, FSymbolListFile : String;
        FStandardCandlesInputFileName : String;
        StandardCandlesInput : TTwoDArray;
        FDumpCandles : TDumpCandles;
        FSkipDumpCandles : Boolean;
        Function AdjustPrices(Bars : TBarList;
                              CorporateActions : TCorporateActions) : TBarList; Overload;
        Function AdjustPrices(Prices : TPriceList;
                              CorporateActions : TCorporateActions) : TPriceList; Overload;
        Procedure SetStandardCandlesInputFileName(Value : String);
        Procedure ProcessSymbolData1Min(Symbol : String;
                                         Bars : TBarList;
                                         AverageDailyVolume : Integer);
        Procedure ProcessSymbolData1Day(Symbol : String;
                                        Bars : TBarList;
                                        Out AverageDailyVolume : Integer);
        Procedure ProcessSymbolFundamental(Symbol : String; Data : TFundamentalData);
        Procedure ProcessSymbolMisc(Symbol : String;
                                    Data1Day : TBarList;
                                    DataHighs, DataLows : TPriceList;
                                    DataFundamental : TFundamentalData;
                                    CorporateActions : TCorporateActions);
        Procedure FindBestCorrelation(Symbol : String;
                                      Data1Day : TBarList;
                                      Out Success : Boolean;
                                      Out MatchesSymbol : String;
                                      Out M, R2 : Double);
        Procedure AVWAP(Symbol : String;
                        DaysBack : Integer;
                        Data1Min : TBarList;
                        Data1Day : TBarList);
        Procedure BuildCharts(Symbol : String;
                              Data1Day : TBarList);

      Public
        Property ChartsDirectory : String Read FChartsDirectory Write FChartsDirectory;
        Property SymbolListFile : String Read FSymbolListFile Write FSymbolListFile;
        Property StandardCandlesInputFileName : String Read FStandardCandlesInputFileName Write SetStandardCandlesInputFileName;
        Property SkipDumpCandles : Boolean Read FSkipDumpCandles Write FSkipDumpCandles;
      End;
    TTrendType = (tOpen, tClose, tHigh, tLow, tGreenBar) ;
Implementation

  Uses
    CandleModels, VolumeWeightedData, SimpleMarketData, GifChart,
    FileNameEncode, BarsFromDatabase,
    DateUtils, SysUtils, Classes, Math;

  /////////////////////////////////////////////////////////////////////////
  // General Candle Routines
  /////////////////////////////////////////////////////////////////////////

  Const
    OneHour = 1.0 / 24;
    Period = OneHour / 4;

  Function ScrubDailyData(Data : TBarList) : TBarList;
    Procedure RemoveZeroDays(Var Data : TBarList);
      Var
        Old, New : Integer;
        NewData : TBarList;
      Begin  { RemoveZeroDays }
        { I've only seen this problem once, but it was enough.  The second to
          last day in the history had a date way in the future, and volume was
          0.  Realtick just ignored that day.  If a stock doesn't trade every
          day, the data just skips the days with no trading, and gives you more
          history. }
        SetLength(NewData, Length(Data));
        New := 0;
        For Old := 0 To Pred(Length(Data)) Do
          If Data[Old].Volume > 0 Then
            Begin
              NewData[New] := Data[Old];
              Inc(New)
            End;
        If (New > 0) And (New <> Length(Data)) Then
          Begin
            { If there was a change, we use the changed data.  If the new data
              has a length of 0 then we have an instrument that doesn't
              report volume ever, so this test doesn't prove anything.  In
              that case we just skip this step of the scrubbing. }
            SetLength(NewData, New);
            Data := NewData
          End
      End;  { RemoveZeroDays }
    Procedure TrimData(Var Data : TBarList);
      Var
        I : Integer;
      Begin  { TrimData }
        I := Pred(Length(Data));
        If I > 0 Then
          Begin
            If (Now - Data[I].StartTime) > 60 Then
              Data := Copy(Data, 0, 0)
            Else Repeat
              If Data[I].Close <= 0 Then
                Begin
                  { I don't know what this means, but it can't be good.
                    I've seen it in some bad cases. }
                  Data := Copy(Data, Succ(I), MaxInt);
                  Break
                End;
              If I = 0 Then
                Exit;
              Dec(I);
              If Data[Succ(I)].StartTime - Data[I].StartTime >= 14 Then
                Begin
                  { This is all too common.  I think that a stock gets delisted,
                    and a new stock reuses the symbol. }
                  Data := Copy(Data, Succ(I), MaxInt);
                  Break
                End
            Until False
          End
      End;  { TrimData }
    Begin  { ScrubDailyData }
      Result := Data;
      RemoveZeroDays(Result);
      TrimData(Result)
    End;  { ScrubDailyData }

  Var
    MarketStart : TDateTime = 6.5;
    Periods : Integer = 26;  // Number of periods during market hours.

  Function GetPeriod(Start : TDateTime) : Integer;
    Begin
      Result := Round((TimeOf(Start) - (MarketStart * OneHour - Period)) / Period)
    End;

  /////////////////////////////////////////////////////////////////////////
  // Correlation Routines
  /////////////////////////////////////////////////////////////////////////

  Const
    MinAcceptableR2 = 0.25;

    // A three month old item will have approximately half the weight
    // of a current item.  This varies with the number of trading
    // days in a month.
    DailyWeight = 1.0083524548945949942911810645439;

  { For each record we find the % change between the open and the close.
    For each date with a record for both lists, we add that point to the
    correlation. }
  Procedure DoOpenCloseCorrelation(ListX, ListY : TBarList;
                                   Out Success : Boolean;
                                   Out M, R2 : Double);
    Var
      X, Y : Double;
      Weight : Double;
      SumXY, SumXX, SumYY : Double;
      IndexX, IndexY : Integer;
      N : Integer;
    Begin
      Success := False;
      Try
        N := 0;
        IndexX := 0;
        IndexY := 0;
        SumXY := 0;
        SumXX := 0;
        SumYY := 0;
        Weight := 1.0;
        Repeat
          If (IndexX >= Length(ListX)) Or (IndexY >= Length(ListY)) Then
            Break;
          If ListX[IndexX].StartTime > ListY[IndexY].StartTime Then
            Inc(IndexY)
          Else If ListX[IndexX].StartTime < ListY[IndexY].StartTime Then
            Inc(IndexX)
          Else
            Begin
              X := (ListX[IndexX].Close - ListX[IndexX].Open) / ListX[IndexX].Open;
              Y := (ListY[IndexY].Close - ListY[IndexY].Open) / ListY[IndexY].Open;
              SumXY := SumXY + X * Y * Weight;
              SumXX := SumXX + X * X * Weight;
              SumYY := SumYY + Y * Y * Weight;
              Inc(N);
              Inc(IndexY);
              Inc(IndexX);
              Weight := Weight * DailyWeight
            End
        Until False;
        If N >= 7 Then
          Begin
            M := SumXY / SumXX;
            R2 := SumXY * SumXY / SumXX / SumYY;
            Success := True
          End
      Except
      End
    End;

  { For each pair of adjacent records we find the % change between the first
    close and the second close.  We only look at pairs if a pair with the
    same dates exists in both lists.  If two lists have the same dates, except
    exactly one day is missing in the middle, two pairs will be ignored from
    the longer list, and one pair from the shorter list. }
  Procedure DoCloseCloseCorrelation(ListX, ListY : TBarList;
                                   Out Success : Boolean;
                                   Out M, R2 : Double);
    Var
      X, Y : Double;
      Weight : Double;
      SumXY, SumXX, SumYY : Double;
      IndexX1, IndexY1, IndexX2, IndexY2 : Integer;
      N : Integer;
    Begin
      Success := False;
      Try
        N := 0;
        IndexX1 := 0;
        IndexY1 := 0;
        SumXY := 0;
        SumXX := 0;
        SumYY := 0;
        Weight := 1.0;
        Repeat
          IndexX2 := Succ(IndexX1);
          IndexY2 := Succ(IndexY1);
          If (IndexX2 >= Length(ListX)) Or (IndexY2 >= Length(ListY)) Then
            Break;
          If ListX[IndexX1].StartTime > ListY[IndexY1].StartTime Then
            Inc(IndexY1)
          Else If ListX[IndexX1].StartTime < ListY[IndexY1].StartTime Then
            Inc(IndexX1)
          Else If ListX[IndexX2].StartTime <> ListY[IndexY2].StartTime Then
            Begin
              Inc(IndexX1);
              Inc(IndexY1)
            End
          Else
            Begin
              X := (ListX[IndexX2].Close - ListX[IndexX1].Close) / ListX[IndexX1].Close;
              Y := (ListY[IndexY2].Close - ListY[IndexY1].Close) / ListY[IndexY1].Close;
              SumXY := SumXY + X * Y * Weight;
              SumXX := SumXX + X * X * Weight;
              SumYY := SumYY + Y * Y * Weight;
              Inc(N);
              Inc(IndexX1);
              Inc(IndexY1);
              Weight := Weight * DailyWeight
            End
        Until False;
        If N >= 7 Then
          Begin
            M := SumXY / SumXX;
            R2 := SumXY * SumXY / SumXX / SumYY;
            Success := True
          End
      Except
      End
    End;

  Procedure TProcessData.FindBestCorrelation(Symbol : String;
                                             Data1Day : TBarList;
                                             Out Success : Boolean;
                                             Out MatchesSymbol : String;
                                             Out M, R2 : Double);
    Var
      I : Integer;
      CurrentSuccess : Boolean;
      CurrentM, CurrentR2 : Double;
    Begin
      Success := False;
      For I := 0 To Pred(Length(PossibleCorrelations)) Do
        If PossibleCorrelations[I].Symbol <> Symbol Then
          Begin
            DoOpenCloseCorrelation(PossibleCorrelations[I].Bars, Data1Day,
                                   CurrentSuccess, CurrentM, CurrentR2);
            If CurrentSuccess Then
              If Not Success Then
                Begin
                  Success := True;
                  MatchesSymbol := PossibleCorrelations[I].Symbol;
                  M := CurrentM;
                  R2 := CurrentR2
                End
              Else If CurrentR2 > R2 Then
                Begin
                  MatchesSymbol := PossibleCorrelations[I].Symbol;
                  M := CurrentM;
                  R2 := CurrentR2
                End
          End
    End;

  Procedure TProcessData.AddFuturesCorrelation(Data1Day : TBarList);
    Begin
      FuturesDailyData := Data1Day
    End;

  Procedure TProcessData.AddPossibleCorrelation(Symbol : String;
                                                Data1Day : TBarList);
    Var
      N : Integer;
    Begin
      N := Length(PossibleCorrelations);
      SetLength(PossibleCorrelations, Succ(N));
      PossibleCorrelations[N].Symbol := Symbol;
      PossibleCorrelations[N].Bars := Data1Day
    End;
  Procedure TProcessData.AVWAP(Symbol : String;
                               DaysBack : Integer;
                               Data1Min : TBarList;
                               Data1Day : TBarList);
   Var
    I, J, StartingDayIndex : Integer;
    BaseDate : TDateTime;
    PV : Double;
    TV : Double;
   Begin
     If Length(Data1Day) > 0 Then
       Begin
         TV := 0.0;
         PV := 0.0;
         // 2 day AVWAP will use the current realtime day and the previous day
         // to calculate, so we align the index with the newest historical date
         // equal to the DaysBack=2 case
         StartingDayIndex := Max(High(Data1Day) - (DaysBack - 2) , 0 );
         BaseDate := Data1Day[StartingDayIndex].StartTime;
         I := 0;
         While I <= High(Data1Min) Do
           Begin
             If Data1Min[I].StartTime >= BaseDate Then
               Break;
               Inc(I);
           End;
         If I <= High(Data1Min) Then
           Begin
             For J := I To High(Data1Min) Do With Data1Min[J] Do
               Begin
                 PV := PV + Volume * (High + Low + Close) / 3.0;
                 TV := TV + Volume;
               End;
           End;
         If PV > 0 Then
           FundamentalData.Add(IntToStr(DaysBack) + ' Day AVWAP PV', Symbol, Format('%g', [PV]));
         If TV > 0 Then
           FundamentalData.Add(IntToStr(DaysBack) + ' Day AVWAP TV', Symbol, Format('%g', [TV]))
       End;
   End; {AVWAP}
  /////////////////////////////////////////////////////////////////////////
  // TProcessData
  /////////////////////////////////////////////////////////////////////////

  Function TProcessData.AdjustPrices(Bars : TBarList;
                        CorporateActions : TCorporateActions) : TBarList;
    Var
      Split, Bar : Integer;
    Begin
      Result := Copy(Bars, 0, MaxInt);
      If Length(CorporateActions.Splits) > 0 Then
        Try
          SetLength(Bars, Length(Bars));
          For Split := Low(CorporateActions.Splits) To High(CorporateActions.Splits) Do
            With CorporateActions.Splits[Split] Do
              For Bar := Low(Result) To High(Result) Do
                With Result[Bar] Do
                  If (StartTime < Date) And (SplitFactor > 0) Then
                    Begin
                      Open := Open / SplitFactor;
                      High := High / SplitFactor;
                      Low := Low / SplitFactor;
                      Close := Close / SplitFactor;
					            Volume := Round(Volume * SplitFactor);
					            PrintCount := Round(PrintCount * SplitFactor);
                      // I believe this is correct for NxCore.  I'm not sure why people adjust the print count,
					            // but they do in fact seem to do it.
                    End
        Except
          SetLength(Result, 0)
        End
    End;
   Function TProcessData.AdjustPrices(Prices : TPriceList;
                        CorporateActions : TCorporateActions) : TPriceList;
    Var
      Split, I : Integer;
    Begin
      Result := Copy(Prices, 0, MaxInt);
      If Length(CorporateActions.Splits) > 0 Then
        Try
          SetLength(Prices, Length(Prices));
          For Split := Low(CorporateActions.Splits) To High(CorporateActions.Splits) Do
            With CorporateActions.Splits[Split] Do
              For I := Low(Result) To High(Result) Do
                If (Result[I].Date < Date) And (SplitFactor > 0) Then
                  Begin
                    Result[I].Price := Result[I].Price / SplitFactor;
                  End
        Except
          SetLength(Result, 0)
        End
      End;


  Procedure TProcessData.SetStandardCandlesInputFileName(Value : String);
    Begin
      If Value <> FStandardCandlesInputFileName Then
        Begin
          FStandardCandlesInputFileName := Value;
          StandardCandlesInput.LoadFromCSV(Value)
        End
    End;

  Procedure TProcessData.ProcessData(Symbol : String;
                                     Data1Min, Data1Day : TBarList;
                                     DataHighs, DataLows : TPriceList;
                                     DataFundamental : TFundamentalData;
                                     CorporateActions : TCorporateActions);
    Procedure DumpBarList(BarList : TBarList);
      Var
        I : Integer;
      Begin  { DumpBarList }
        For I := Low(BarList) To High(BarList) Do
          With BarList[I] Do
            MultiWriteLn('  ' + IntToStr(I) +
                    ':  (Open: ' + FloatToStr(Open) +
                    ', High: ' + FloatToStr(High) +
                    ', Low: ' + FloatToStr(Low) +
                    ', Close: ' + FloatToStr(Close) +
                    ', StartTime: ' + DateTimeToStr(StartTime) +
                    ', Volume: ' + IntToStr(Volume) +
                    ')')
      End;  { DumpBarList }
    Procedure DumpPriceList(PriceList : TPriceList);
      Var
        I : Integer;
      Begin  { DumpPriceList }
        For I := Low(PriceList) To High(PriceList) Do
          With PriceList[I] Do
            MultiWriteLn('  ' + IntToStr(I) +
                    ':  (Price: ' + FloatToStr(Price) +
                    ', Date: ' + DateTimeToStr(Date) +
                    ')')
      End;  { DumpPriceList }
    Var
      Status : String;
      AverageDailyVolume : Integer;
    Begin  { TProcessData.ProcessData }
      Try
        MultiWriteLn('Processing:  ' + Symbol);
        Status := 'Adjusting Prices';
        Data1Min := AdjustPrices(Data1Min, CorporateActions);
        Data1Day := AdjustPrices(Data1Day, CorporateActions);
        Status := 'ScrubDailyData';
        Data1Day := ScrubDailyData(Data1Day);
        Status := 'ProcessSymbolData1Day';
        ProcessSymbolData1Day(Symbol, Data1Day, AverageDailyVolume);
        Status := 'ProcessSymbolData1Min';
        ProcessSymbolData1Min(Symbol, Data1Min, AverageDailyVolume);
        Status := 'ProcessSymbolFundamental';
        ProcessSymbolFundamental(Symbol, DataFundamental);
        Status := 'ProcessSymbolMisc';
        ProcessSymbolMisc(Symbol, Data1Day, DataHighs, DataLows, DataFundamental, CorporateActions);
        Status := 'AVWAP';
        AVWAP(Symbol, 2, Data1Min, Data1Day);
        AVWAP(Symbol, 3, Data1Min, Data1Day);
        AVWAP(Symbol, 4, Data1Min, Data1Day);
        AVWAP(Symbol, 5, Data1Min, Data1Day);
        Status := 'BuildCharts';
        If ChartsDirectory <> '' Then
          BuildCharts(Symbol, Data1Day);
        Status := 'DumpCandles';
        If Not SkipDumpCandles Then
          FDumpCandles.AddData(Symbol, Data1Min, Data1Day, CorporateActions)
      Except
        On E : Exception Do
          Begin
            MultiWriteLn(E.ClassName + ':  ' + E.Message);
            If E Is EExternal Then
              MultiWriteLn('ExceptionAddress:' + IntToHex(Integer((E As EExternal).ExceptionRecord^.ExceptionAddress),8));
            MultiWriteLn('Exception in TProcessData.ProcessData(' + Symbol + '), ' + Status);
            //MultiWriteLn('Data1Min:');
            //DumpBarList(Data1Min);
            //MultiWriteLn('Data1Day:');
            //DumpBarList(Data1Day);
            //MultiWriteLn('DataHighs:');
            //DumpPriceList(DataHighs);
            //MultiWriteLn('DataLows:');
            //DumpPriceList(DataLows);
            MultiWriteLn('CompanyName: "' + DataFundamental.CompanyName + '", ListedExchange: "' + DataFundamental.ListedExchange + '"');
            Sleep(100);
          End
      End
    End;  { TProcessData.ProcessData }

  Procedure TProcessData.ProcessSymbolFundamental(Symbol : String; Data : TFundamentalData);
    Var
      FinalListedExchange : String;
    Procedure CopyField(Const Name, Value : String);  Overload;
      Begin  { CopyField String }
        If Value <> '' Then
          FundamentalData.Add(Name, Symbol, Value)
      End;
    Procedure CopyField(Const Name : String; Value : Integer);  Overload;
      Begin  { CopyField Integer }
        If Value > 0 Then
          FundamentalData.Add(Name, Symbol, IntToStr(Value))
      End;
    Procedure CopyField(Const Name : String; Value : Double);  Overload;
      Begin  { CopyField Double }
        If Value <> 0.0 Then
          FundamentalData.Add(Name, Symbol, Format('%g', [Value]))
      End;
    Procedure CopyFieldWithZero(Const Name : String; Value : Double);
      Begin  { CopyFieldWithZero Double }
        FundamentalData.Add(Name, Symbol, Format('%g', [Value]))
      End;
    Begin  { TProcessData.ProcessSymbolFundamental }
      With Data Do
        Begin
          CopyField('Company Name', CompanyName);
          // The older (Windows / Delphi / Realtick) server would read this
          // field from the data feed, so we stored it in the F_ file.  The
          // newer (Linux / C++ / Proxy) server reads it from OvernightData.csv.
          // For now we put it in both places.
          FinalListedExchange := ListedExchange;
          If ListedExchange <> '' Then
            Begin
              If (ListedExchange = 'PINK') Or (ListedExchange = 'OTC') Then
                Begin
                  If Copy(AlternateExchange, 1, 5) = 'OTCQX' Then
                    FinalListedExchange := 'OQX'
                  Else If Copy(AlternateExchange, 1, 5) = 'OTCQB' Then
                    FinalListedExchange := 'OQB'
                  Else
                    FinalListedExchange := 'PINK'
                End
            End;
          OutputData.Add('Listed Exchange', Symbol, FinalListedExchange);
          CopyField('Listed Exchange', FinalListedExchange);
          //CopyField('Alternate Exchange', AlternateExchange);
          CopyField('Prev Put Volume', PutVolume);
          CopyField('Prev Call Volume', CallVolume);
          CopyField('Dividend', Dividend);
          CopyField('Forward Dividend Rate', ForwardDividendRate);
          CopyField('Trailing Dividend Rate', TrailingDividendRate);
          CopyField('Earnings', Earnings);
          CopyField('P/E Ratio', PERatio);
          CopyField('Beta', Beta);
          CopyField('Market Cap', MarketCap);
          CopyField('Shares Outstanding', SharesOutstanding);
          CopyField('EPS Net Income', EpsNetIncome);
          CopyField('Income', Income);
          CopyField('Assets', Assets);
          CopyFieldWithZero('Debt', Debt);
          CopyField('Short Interest', ShortInterest);
          CopyField('NAICS Code', NaicsCode);
          CopyField('NAICS', Naics);
          CopyField('CUSIP', Cusip);
          CopyField('EnterpriseValue', EnterpriseValue);
          CopyField('Revenue', Revenue);
          CopyField('ShortGrowth', ShortGrowth);
          CopyField('QuarterlyRevenueGrowth', QuarterlyRevenueGrowth);
          CopyField('QuarterlyEarningsGrowth', QuarterlyEarningsGrowth);
          CopyField('PercentHeldByInsiders', PercentHeldByInsiders);
          CopyField('PercentHeldByInstitutions', PercentHeldByInstitutions);
          CopyField('ShortRatio', ShortRatio);
          CopyField('ShortPercentOfFloat', ShortPercentOfFloat);
          CopyField('Cash', TotalCash);
          CopyField('EstimatedQuarterlyEarningsGrowth', EstimatedQuarterlyEarningsGrowth);
          CopyField('EstimatedAnnualEarningsGrowth', EstimatedAnnualEarningsGrowth);
          CopyField('PEGRatio', PEGRatio);
          CopyField('Float', Float);
          CopyField('DaysToCover', DaysToCover);
          CopyField('Profile', Profile)
        End
    End;  { TProcessData.ProcessSymbolFundamental }

  Var
    { Just for debugging. }
    RealDate : String;

  Function CompareDoubles(Item1, Item2: Pointer): Integer;
    Var
      D1, D2 : Double;
    Begin
      D1 := PDouble(Item1)^;
      D2 := PDouble(Item2)^;
      If D1 < D2 Then
        Result := -1
      Else If D1 > D2 Then
        Result := 1
      Else
        Result := 0
    End;

  Procedure TProcessData.ProcessSymbolData1Min(Symbol : String;
                                               Bars : TBarList;
                                               AverageDailyVolume : Integer);
    Var
      StartTimeMinutes : Integer;
      MarketOpenMinutes : Integer;
    Function ExtractFrom1MinuteBars(MinutesPerBar : Byte;
                                    OneMinuteBars : TBarList) : TBarList;
      { MinutesPerBar described the output.  If this is two, then adjacent pairs of
          one minute bars are merged.  We always start counting with the first bar
          at the open.  If the number of minutes per bar does not divide evenly into
          the number of bars in a day, then the last bar gets shorted.  We divide
          up the day based only on the MinutesPerBar input, not the actual bar data.
          If a minute is missing from the day, then we just have fewer minutes in
          the resulting bar in the output.  For example, if MinutesPerBar=3 and
          there were no trades in the second minute, then we use the first two bars
          of input to create the first bar of output.  The third bar of input will
          apply to the second bar of input.
        OneMinuteBars in the input.  We always start from one minute bars for
          simplicity.  Presumably we could use 5 minute bars to make 10 and 15
          minute bars.  But since we are only makeing one request, and that request
          is in a different piece of code, we always request one minute bars.  The
          datasource will sometimes skip a bar by leaving the bar out, and other
          times it will skip the bar by setting volume to 0.  Other times a symbol
          will have no volume at all.  This function deals with all of these cases
          correctly.
        If a period has no data, we deal with that in the most naive way.  If all
          of the input bars corresponding to this period had a volume of 0, we
          report a volume of 0 on the output bar.  If all of the input bars
          corresponding to an output bar are missing, then the output bar will be
          missing. }
      Var
        InputIndex, OutputIndex : Integer;
        LastOutputDate, CurrentInputDate : TDateTime;
        LastOutputPeriod, CurrentInputPeriod, CurrentInputMinute : Integer;
      Begin  { ExtractFrom1MinuteBars }
        Assert(MinutesPerBar > 0);
        SetLength(Result, Length(OneMinuteBars));
        LastOutputDate := 0;
        LastOutputPeriod := -1;
        OutputIndex := -1;
        For InputIndex := 0 To Pred(Length(OneMinuteBars)) Do
          Begin
            CurrentInputDate := DateOf(OneMinuteBars[InputIndex].StartTime);
            CurrentInputMinute := MinuteOfTheDay(OneMinuteBars[InputIndex].StartTime);
            If (CurrentInputMinute < StartTimeMinutes) Or
               (CurrentInputMinute >=
                (StartTimeMinutes + MarketOpenMinutes)) Or
               (DayOfTheWeek(CurrentInputDate) In [DaySaturday, DaySunday]) Then
              { Skip a bar that is not part of a normal trading day.  This
                unfortunately misses some things, like when normal stocks
                are on holiday, but the futures are still trading.  In that
                case we would ideally like to skip the day for the futures,
                since the data is so sparse. }
              CurrentInputPeriod := -1
            Else
              CurrentInputPeriod :=
                (CurrentInputMinute - StartTimeMinutes) Div MinutesPerBar;
            If CurrentInputPeriod < 0 Then
              Begin
                { Skip it. }
              End
            Else If (CurrentInputDate = LastOutputDate) And
                    (CurrentInputPeriod = LastOutputPeriod) Then
              With OneMinuteBars[InputIndex] Do
                Begin
                  { Add to this candle. }
                  Result[OutputIndex].High := Max(Result[OutputIndex].High, High);
                  Result[OutputIndex].Low := Min(Result[OutputIndex].Low, Low);
                  Result[OutputIndex].Close := Close;
                  Result[OutputIndex].Volume := Result[OutputIndex].Volume + Volume
                End
            Else
              Begin
                { Start a new candle. }
                Inc(OutputIndex);
                Result[OutputIndex] := OneMinuteBars[InputIndex];
                Result[OutputIndex].StartTime := IncMinute(DateOf(Result[OutputIndex].StartTime), CurrentInputPeriod * MinutesPerBar + StartTimeMinutes)
              End;
            LastOutputDate := CurrentInputDate;
            LastOutputPeriod := CurrentInputPeriod
          End;
        SetLength(Result, Succ(OutputIndex))
      End;  { ExtractFrom1MinuteBars }
    Procedure RemoveZeroVolumeBars(Var Bars : TBarList);
      Var
        StockContainsVolume : Boolean;
        Src, Dest : Integer;
      Begin
        StockContainsVolume := False;
        For Src := Low(Bars) To High(Bars) Do
          If Bars[Src].Volume > 0 Then
            Begin
              StockContainsVolume := True;
              Break
            End;
        If Not StockContainsVolume Then
          // Some stocks don't contain any volume, so we can't use this test.
          // Otherwise, a bar with no volume is a missing bar.  Note:  We are
          Exit;
        // Note:  We are purposely trying to fool BreakAtMissingBar.  We want it
        // to ignore bars which are missing at the beginning or end of the day.
        // We need this because of the half days when a lot of data is missing.
        // This is not an ideal solution, since it doesn't work pefectly on
        // other days, but it should be close enough.  In any case, it's
        // better than we had before when the half days really screwed us.
        Dest := Low(Bars);
        For Src := Low(Bars) To High(Bars) Do
          If Bars[Src].Volume > 0 Then
            Begin
              Bars[Dest] := Bars[Src];
              Inc(Dest)
            End;
        SetLength(Bars, Dest - Low(Bars))
      End;
    Procedure BreakAtMissingBar(Var Bars : TBarList; MinutesPerCandle : Byte);
      { This takes bars which came from the output of ExtractFrom1MinuteBars
        and it looks for missing bars.  If there is a missing bar, then we throw
        away all the data before it.  If there is more than one, then we throw
        away everything before the last one. }
      Var
        PreviousDate, CurrentDate : TDateTime;
        PreviousMinutes, CurrentMinutes : Integer;
        I : Integer;
      Begin  { BreakAtMissingBar }
        RemoveZeroVolumeBars(Bars);
        Exit;
        PreviousMinutes := 0;
        PreviousDate := 0;
        For I := High(Bars) DownTo Low(Bars) Do
          Begin
            CurrentDate := DateOf(Bars[I].StartTime);
            CurrentMinutes := MinuteOfTheDay(Bars[I].StartTime);
            If (CurrentDate = PreviousDate) And
               (CurrentMinutes + MinutesPerCandle <> PreviousMinutes) Then
              Begin
                { The bars are not consecutive.  We work backwards from the
                  most recent bar and stop when we get to a missing bar.  The
                  logic can miss some cases, like a stock which is perfect
                  except that the first or last bar of a day is missing.  But
                  the logic should do a sufficient job most of the time. }
                Bars := Copy(Bars, Succ(I), MaxInt);
                Exit
              End;
            PreviousDate := CurrentDate;
            PreviousMinutes := CurrentMinutes
          End
      End;  { BreakAtMissingBar }
    Procedure SaveStandardCandles;
      Var
        EncodedBars, NewBar : String;
        WhichCandleSet, MinutesPerCandle, MaxCandleCount, CurrentCandleCount : Integer;
        CurrentCandleSet : TBarList;
        AllCandleSets : TStrings;
        MinutesPerCandleKey : String;
        I : Integer;
      Begin  { SaveStandardCandles }
        SetLength(CurrentCandleSet, 0);  { Silly Compiler Warning. }
        AllCandleSets := TStringList.Create;
        Try
          StandardCandlesInput.GetRowHeaders(AllCandleSets);
          For WhichCandleSet := 0 To Pred(AllCandleSets.Count) Do
            Begin
              MinutesPerCandleKey := AllCandleSets[WhichCandleSet];
              MinutesPerCandle := StrToIntDef(MinutesPerCandleKey, -1);
              Assert((MinutesPerCandle > 0) And (MinutesPerCandle <= High(Byte)));
              MaxCandleCount := StrToIntDef(StandardCandlesInput.Get('Max Candle Count', MinutesPerCandleKey), -1);
              If MaxCandleCount < 1 Then
                MaxCandleCount := MaxInt;
              CurrentCandleSet := ExtractFrom1MinuteBars(MinutesPerCandle, Bars);
              BreakAtMissingBar(CurrentCandleSet, MinutesPerCandle);
              CurrentCandleCount := 0;
              EncodedBars := '';
              For I := High(CurrentCandleSet) DownTo Low(CurrentCandleSet) Do
                Begin
                  With CurrentCandleSet[I] Do
                    NewBar := Format('%g:%g:%g:%g:%d',
                                     [Open, High, Low, Close, Volume]);
                  If CurrentCandleCount = 0 Then
                    EncodedBars := NewBar
                  Else
                    EncodedBars := NewBar + ';' + EncodedBars;
                  Inc(CurrentCandleCount);
                  If CurrentCandleCount >= MaxCandleCount Then
                    Break
                End;
              { Save all of them, even the empties, so we get our header row
                correct. }
              StandardCandles.Add(MinutesPerCandleKey, Symbol, EncodedBars)
            End
        Finally
          AllCandleSets.Free
        End
      End;  { SaveStandardCandles }
    Procedure MakeVolumeCandles(VolumeBreak : Integer; Bars : TBarList);
      Const
        BarTime = 1.0 / 24.0 / 4.0;
      Var
        Candles : TCandleModelList;
        VolumeBlockFactory : TVolumeBlockFactory;
        VolumeRemainingThisCandle, I : Integer;
        Prints : TPrints;
      Begin  { MakeVolumeCandles }
        Assert(VolumeBreak > 0);
        Candles := TCandleModelList.Create;
        VolumeBlockFactory := TVolumeBlockFactory.Create(VolumeBreak);
        Try
          For I := Low(Bars) To High(Bars) Do
            With Bars[I] Do
              If StartTime > Now - 7 Then
                Candles.AddCandle(TCandleModel.Create(Open, High, Low, Close, Volume, StartTime, StartTime + BarTime));
          VolumeRemainingThisCandle := VolumeBreak;
          While Not Candles.Done Do
            Begin
              If VolumeRemainingThisCandle = 0 Then
                VolumeRemainingThisCandle := VolumeBreak;
              Candles.ExtractVolume(VolumeRemainingThisCandle, Prints);
              For I := Low(Prints) To High(Prints) Do
                With Prints[I] Do
                  VolumeBlockFactory.AddPrint(Price, Volume, Time)
            End;
          VolumeBlocks.Add('Vol', Format('%s_%d', [Symbol, VolumeBreak]), VolumeBlocksToString(VolumeBlockFactory.GetBlocks, True));
        Finally
          Candles.Free;
          VolumeBlockFactory.Free;
        End
      End;  { MakeVolumeCandles }
    Procedure FindBunnies(MinutesPerBar : Byte);
      Const
        Periods = 23;
      Var
        Key : String;
        CurrentCandleSet : TBarList;
        SX, SXX, SY, SYY, SXY : Double;  { X is time, Y is price. }
        StdDev, M, B : Double;
        I : Integer;
        Differences : Array [0..(Periods-1)] Of Double;
        Value : Double;
      Begin  { FindBunnies }
        Key := 'Bunny ' + IntToStr(MinutesPerBar);
        FundamentalData.Add(Key, Symbol, '');  // Reserve space in the header even if we print nothing.
        CurrentCandleSet := ExtractFrom1MinuteBars(MinutesPerBar, Bars);
        BreakAtMissingBar(CurrentCandleSet, MinutesPerBar);
        If Length(CurrentCandleSet) >= Periods Then
          Try
            CurrentCandleSet := Copy(CurrentCandleSet, Length(CurrentCandleSet) - Periods, Periods);
            SX := 0;
            SXX := 0;
            SY := 0;
            SYY := 0;
            SXY := 0;
            For I := Low(Differences) To High(Differences) Do
              Begin
                SX := SX + I;
                SXX := SXX + Sqr(I);
                SY := SY + CurrentCandleSet[I].Close;
                SYY := SYY + Sqr(CurrentCandleSet[I].Close);
                SXY := SXY + I * CurrentCandleSet[I].Close
              End;
            StdDev := Sqrt((Periods * SYY - Sqr(SY)) / (Periods * Pred(Periods)));
            M := (SXY - SX * SY / Periods) / (SXX - SX * SX / Periods);
            B := (SY - M * SX) / Periods;
            For I := Low(Differences) To High(Differences) Do
              Differences[I] := (CurrentCandleSet[I].Close - (M * I + B)) / StdDev;
            Value := Sqrt(SumOfSquares(Differences) / Periods);
            FundamentalData.Add(Key, Symbol, FloatToStrF(Value, ffGeneral, 6, 0))
          Except
            // If everything is flat, std dev will be 0, and we'll have a
            // divided by 0 error.
          End
      End;  { FindBunnies }
    Procedure Find5MinAverageSize;
      // This value is required by the SMB radar.  It serves the same general
      // purpose as the ATR or the volume weighted volatility.  In fact, when
      // we used the ATR as a placeholder, we got almost identical rankings,
      // but different values.  The values actually matter in some cases.  In
      // particular, the "Price Spike" column should be yellow if it is greater
      // than 5.  So we try to reproduce their number as accurately as possible.
      Const
        MinutesPerBar = 5;
        // The spec said to look at all the 5 day bars in the last 20 days.  We
        // assume for simplicity that we have the ideal case.  6.5 hours/day *
        // 20 days * 60 minutes/hour / (5 minutes/bar) = 1560 bars.
        IdealCount = 1560;
      Var
        Key : String;
        CurrentCandleSet : TBarList;
        TotalRange : Double;
        BarCount, I : Integer;
        Value : Double;
      Begin  { Find5MinAverageSize }
        Key := 'Average Bar Size ' + IntToStr(MinutesPerBar);
        FundamentalData.Add(Key, Symbol, '');  // Reserve space in the header even if we print nothing.
        CurrentCandleSet := ExtractFrom1MinuteBars(MinutesPerBar, Bars);
        // I am (somewhat naively) going to keep the 0 volume bars and
        // everything before them.  That probably makes some sense here.  If a
        // stock has 0 volume bars, we should lower the average volatility.
        // (Notice the previous comment, we don't know for certain if a 0
        // volume bar would be passed to us, or would show up as a missing bar.)
        // Also, since this is just an average, it makes some sense to try to
        // use as much data is available.
        // BreakAtMissingBar(CurrentCandleSet, MinutesPerBar);
        //
        // We want 20 days worth of 5 minute bars.
        // 20 days * 390 minutes/day / 5 minutes/bar
        BarCount := Min(20 * 390 Div MinutesPerBar, Length(CurrentCandleSet));
        If BarCount > 0 Then
          Begin
            CurrentCandleSet := Copy(CurrentCandleSet, Length(CurrentCandleSet) - BarCount, BarCount);
            TotalRange := 0.0;
            For I := Low(CurrentCandleSet) To High(CurrentCandleSet) Do
              With CurrentCandleSet[I] Do
                TotalRange := TotalRange + (High - Low);
            Value := TotalRange / BarCount;
            FundamentalData.Add(Key, Symbol, FloatToStrF(Value, ffGeneral, 6, 0))
          End
      End;  { Find5MinAverageSize }
    Procedure SharesPerPrint;
      Var
        I : Integer;
        TotalShares, TotalPrints : Double;
      Begin  { SharesPerPrint }
        TotalShares := 0;
        TotalPrints := 0;
        For I := Low(Bars) To High(Bars) Do
          With Bars[I] Do
            Begin
              TotalShares := TotalShares + Volume;
              TotalPrints := TotalPrints + PrintCount
            End;
        If (TotalShares > 0) And (TotalPrints > 0) Then
          OutputData.Add('Shares per Print', Symbol, FloatToStrF(TotalShares / TotalPrints, ffGeneral, 6, 0))
      End;  { SharesPerPrint }
    Var
      TotalDays : Integer;
      MarketVolume : Int64;  // During market hours.
      PreVolume : Int64;  // Before market hours.
      PostVolume : Int64;  // After market hours.
      PreviousTimeTag : TDateTime;
      I : Integer;
      SumOfVolumeWeightedChanges : Double;
      VolumeBreak : Integer;
      VolumeByPeriod : Array Of Integer;
      TimeFrame : Integer;
    Begin  { TProcessData.ProcessSymbolData1Min }
      // this initialization is kind of ugly, changing Delphi constants to variables
      // for the market hours is giving me fits.
      SetLength(VolumeByPeriod, Succ(Periods));
      StartTimeMinutes := Round(MarketStart * 60);  { Start at 6:30am }
      MarketOpenMinutes := Round(Periods * 15);  { The market is open for 6 1/2 hours. }

      SaveStandardCandles;
      FindBunnies(130);
      Find5MinAverageSize;
      SharesPerPrint;
      TotalDays := 0;
      MarketVolume := 0;
      PreVolume := 0;
      PostVolume := 0;
      PreviousTimeTag := 0;
      SumOfVolumeWeightedChanges := 0;
      For I := Low(Bars) To High(Bars) Do
        With Bars[I] Do
          Begin
            RealDate := DateTimeToStr(StartTime);
            TimeFrame := GetPeriod(StartTime);
            If TimeFrame < 1 Then
              Inc(PreVolume, Volume)
            Else If TimeFrame > Periods Then
              Inc(PostVolume, Volume)
            Else
              Begin
                If DateOf(PreviousTimeTag) <> DateOf(StartTime) Then
                  Inc(TotalDays);
                Inc(MarketVolume, Volume);
                PreviousTimeTag := StartTime;
                SumOfVolumeWeightedChanges := SumOfVolumeWeightedChanges + (High - Low) * Sqrt(Volume)
              End
          End;
      If MarketVolume > 0 Then
        Begin
          VolumeBreak := Round(MarketVolume / TotalDays / Periods);
          OutputData.Add('Volume Break', Symbol, Format('%d', [VolumeBreak]));
          OutputData.Add('Tick Volatility', Symbol, Format('%g', [SumOfVolumeWeightedChanges * Sqrt(VolumeBreak) / MarketVolume]));
          For I := 1 To High(VolumeByPeriod) Do
            VolumeByPeriod[I] := 0;
          For I := Low(Bars) To High(Bars) Do
            With Bars[I] Do
              Begin
                TimeFrame := Round((TimeOf(StartTime) - (MarketStart * OneHour - Period)) / Period);
                If (TimeFrame >= 1) And (TimeFrame <= System.High(VolumeByPeriod)) Then
                  Inc(VolumeByPeriod[TimeFrame], Volume)
              End;
          For I := 1 To High(VolumeByPeriod) Do
            VolumeData.Add(Format('%d', [I]), Symbol, Format('%d', [Round(VolumeByPeriod[I]/TotalDays)]));
          VolumeData.Add('Pre', Symbol, Format('%d', [Round(PreVolume/TotalDays)]));
          VolumeData.Add('Post', Symbol, Format('%d', [Round(PostVolume/TotalDays)]));
          If ((AverageDailyVolume >= 150000) Or SymbolIsFuture(Symbol)) And (VolumeBreak > 0) Then
            MakeVolumeCandles(VolumeBreak, Bars);
        End
    End;  { TProcessData.ProcessSymbolData1Min }

  Function CommonVolatility(Bars : TBarList; Days : Integer) : Double;
    Var
      Values : Array Of Double;
      Mean, Sum : Double;
      I : Integer;
    Begin
      Result := 0;
      If Days < Length(Bars) Then
        Try
          SetLength(Values, Days);
          For I := 1 To Days Do
            Values[Pred(I)] := Ln(Bars[Length(Bars)-I].Close / Bars[Pred(Length(Bars) - I)].Close);
          Sum := 0;
          For I := 0 To Pred(Days) Do
            Sum := Sum + Values[I];
          Mean := Sum / Days;
          Sum := 0;
          For I := 0 To Pred(Days) Do
            Sum := Sum + Sqr(Values[I] - Mean);
          Result := Sqrt(Sum * 252 / Days)
        Except
        End
    End;

  Function CommonVolatilityMonths(Bars : TBarList; Months : Integer) : Double;
    Var
      FirstDate : TDateTime;
      I : Integer;
    Begin
      Result := 0;
      If Length(Bars) > 0 Then
        Begin
          FirstDate := Bars[Pred(Length(Bars))].StartTime;
          FirstDate := IncMonth(FirstDate, -Months);
          FirstDate := Round(FirstDate);  { Try to avoid the problems around daylight savings time. }
          I := 0;
          { A binary search might save some time. }
          While (I < Pred(Length(Bars))) And (Round(Bars[I].StartTime) <= FirstDate) Do
            Inc(I);
          If I > 0 Then
            { We must skip at least one day, or we can't be sure this is valid.
              If we have only 5 days of data, and we use this algorithm to go
              back a month, we will think that we have an entire month's data.
              We need to see at least one value before the month started to prove
              to ourselves that we have an entire months data.  If we have exactly
              the amount of data requested, we will fail.  It's hard to do that
              better. }
            Result := CommonVolatility(Bars, Length(Bars) - I)
        End
    End;

  { This procedure has to merge different types of data.  To deliver the 52 week
    high, we ask TAL for the 52 week high, and we look at the daily data
    ourselves.  I don't remember the specific problems, but neither of these
    was sufficient by itself.

    For the lifetime high and low we do something similar.  We start with the
    monthly data, because it goes back so far.  But then we look at the 52 week
    high/low that we just computed.  I don't think that the monthly numbers
    always include the current month.  Or maybe part of the month, but just not
    the last day.  It seems to be inconsistant.

    All of this logic assumes that all the prices will be greater than 0.  This
    does not work for the advance decline line and similar values. :( }
  Procedure TProcessData.ProcessSymbolMisc(Symbol : String;
                                           Data1Day : TBarList;
                                           DataHighs, DataLows : TPriceList;
                                           DataFundamental : TFundamentalData;
                                           CorporateActions : TCorporateActions);
    Var
      XWeekHigh, XWeekLow : Double;
      YHigh, YLow : Double;
    Procedure XWeekHighLow(Weeks : Integer);
      Var
        I : Integer;
        EarliestDate : TDateTime;
      Begin  { YearHighLow }
        If Length(Data1Day) > 0 Then
          Begin
            With Data1Day[Pred(Length(Data1Day))] Do
              Begin
                EarliestDate := IncDay(IncWeek(StartTime, -1 * Weeks));
                XWeekHigh := High;
                XWeekLow := Low
              End;
            For I := 0 To Length(Data1Day) - 2 Do
              With Data1Day[I] Do
                If StartTime >= EarliestDate Then
                  Begin
                    XWeekHigh := Max(XWeekHigh, High);
                    XWeekLow := Min(XWeekLow, Low)
                  End
          End;
        If XWeekHigh > 0 Then
          FundamentalData.Add(IntToStr(Weeks) + ' Week High', Symbol, Format('%g', [XWeekHigh]));
        If XWeekLow > 0 Then
          FundamentalData.Add(IntToStr(Weeks) + ' Week Low', Symbol, Format('%g', [XWeekLow]))
      End;  { YearHighLow }
    Procedure YearHighLow;
      Var
        I : Integer;
        EarliestDate : TDateTime;
      Begin  { YearHighLow }
        If Length(Data1Day) > 0 Then
          Begin
            With Data1Day[Pred(Length(Data1Day))] Do
              Begin
                EarliestDate := IncDay(IncYear(StartTime, -1));
                YHigh := High;
                YLow := Low
              End;
            For I := 0 To Length(Data1Day) - 2 Do
              With Data1Day[I] Do
                If StartTime >= EarliestDate Then
                  Begin
                    YHigh := Max(YHigh, High);
                    YLow := Min(YLow, Low)
                  End
          End;
        YHigh := Max(YHigh, DataFundamental.High52Week);
        If DataFundamental.Low52Week > 0 Then
          YLow := Min(YLow, DataFundamental.Low52Week);
        If YHigh > 0 Then
          FundamentalData.Add('52 Week High', Symbol, Format('%g', [YHigh]));
        If YLow > 0 Then
          FundamentalData.Add('52 Week Low', Symbol, Format('%g', [YLow]))
      End;  { YearHighLow }
    Procedure TwoYearHighLow;
      Var
        I : Integer;
        EarliestDate : TDateTime;
        TwoYHigh, TwoYLow : Double;
      Begin  { TwoYearHighLow }
        TwoYHigh := 0;
        TwoYLow := 0;
        If Length(Data1Day) > 0 Then
          Begin
            With Data1Day[Pred(Length(Data1Day))] Do
              Begin
                EarliestDate := IncDay(IncYear(StartTime, -2));
                TwoYHigh := High;
                TwoYLow := Low
              End;
            For I := 0 To Length(Data1Day) - 2 Do
              With Data1Day[I] Do
                If StartTime >= EarliestDate Then
                  Begin
                    TwoYHigh := Max(TwoYHigh, High);
                    TwoYLow := Min(TwoYLow, Low)
                  End
          End;
        TwoYHigh := Max(TwoYHigh, DataFundamental.High52Week);
        If DataFundamental.Low52Week > 0 Then
          TwoYLow := Min(TwoYLow, DataFundamental.Low52Week);
        If TwoYHigh > 0 Then
          FundamentalData.Add('104 Week High', Symbol, Format('%g', [TwoYHigh]));
        If TwoYLow > 0 Then
          FundamentalData.Add('104 Week Low', Symbol, Format('%g', [TwoYLow]))
      End;  { TwoYearHighLow }
    Procedure LifetimeHighLow;
      Function SafeMin(Var Accumulator : Double; NewValue : Double) : Boolean;
        Begin  { SafeMin }
          { Assume 0 means no value.  Assume we will never get a value less than
            0.  Sometimes TAL gives us 0 for no good reason.  The most correct
            answer when we see a value of 0 would be to say that we don't know,
            the final answer.  But it seems like just ignoring this value will do
            a decent job of giving us a result.  The result will be correct in
            some cases that I've seen, and probably pretty close in most other
            real cases. }
          If (Accumulator <= 0)
             Or ((NewValue > 0) And (NewValue <= Accumulator)) Then
            Begin
              Accumulator := NewValue;
              Result := True
            End
          Else
            Result := False
        End;  { SafeMin }
      Var
        LHigh, LLow : Double;
        AdjustedHighs, AdjustedLows: TPriceList;
        I : Integer;
        HighestPriceIndex, LowestPriceIndex : Integer;
        NewestBarOrSplitDate : TDateTime;
      Begin  { LifetimeHighLow }
        { The low is the lowest value that we see in the lows database or the the
          yearly low, whichever is lower.  The high is the highest value that we
          see in the highs database or the yearly high, whichever is higher.  0
          for the yearly or lifetime high or low means that we don't know the
          value. }
        NewestBarOrSplitDate := 0;
        If Length(CorporateActions.Splits) > 0 Then
          NewestBarOrSplitDate := CorporateActions.Splits[Pred(Length(CorporateActions.Splits))].Date;
        If Length(Data1Day) > 0 Then
          NewestBarOrSplitDate := Max(NewestBarOrSplitDate, Data1Day[Pred(Length(Data1Day))].StartTime);

        LHigh := YHigh;
        LLow := YLow;
        HighestPriceIndex := -1;
        LowestPriceIndex := -1;
        AdjustedHighs := AdjustPrices(DataHighs, CorporateActions);
        AdjustedLows := AdjustPrices(DataLows, CorporateActions);
        If Length(DataHighs) > 0 Then
          Begin
            If (Length(AdjustedHighs) > 0) Then
              Begin
                For I := 0 To Pred(Length(AdjustedHighs)) Do
                  If AdjustedHighs[I].Price > LHigh Then
                    Begin
                      LHigh := AdjustedHighs[I].Price;
                      HighestPriceIndex := I
                    End
              End
          End;
        If Length(DataLows) > 0 Then
          Begin
            If (Length(AdjustedLows) > 0) Then
              Begin
                For I := 0 To Pred(Length(AdjustedLows)) Do
                  If SafeMin(LLow, AdjustedLows[I].Price) Then
                    Begin
                      LowestPriceIndex := I;
                    End
              End
          End;

        { I ran into a problem where there was a reverse split two days after the last
          print for a Pink Sheets symbol.  The high and low were recorded with the wrong
          date.  They were listed before the split because I was looking at the last
          print.  Now we take the later of the last split and the last print when
          inserting.  This should end up working even if the split is reported late,
          though the daily alerts will be wrong until the split is reported and the data
          gets fixed }
        If LHigh > 0 Then
          Begin
            FundamentalData.Add('Lifetime High', Symbol, Format('%g', [LHigh]));
            If (HighestPriceIndex <> -1) Then
              UpdateDatabaseHigh(Symbol, DataHighs[HighestPriceIndex].Price, DataHighs[HighestPriceIndex].Date)
            Else If NewestBarOrSplitDate > 0 Then
              UpdateDatabaseHigh(Symbol, LHigh, NewestBarOrSplitDate)
          End;
        If LLow > 0 Then
          Begin
            FundamentalData.Add('Lifetime Low', Symbol, Format('%g', [LLow]));
            If (LowestPriceIndex <> -1) Then
              UpdateDatabaseLow(Symbol, DataLows[LowestPriceIndex].Price, DataLows[LowestPriceIndex].Date)
            Else If NewestBarOrSplitDate > 0 Then
              UpdateDatabaseLow(Symbol, LLow, NewestBarOrSplitDate)
          End;
      End;  { LifetimeHighLow }
    Begin  { TProcessData.ProcessSymbolMisc }
      YHigh := 0;
      YLow := 0;
      XWeekHigh := 0;
      XWeekLow := 0;
      XWeekHighLow(13);
      XWeekHighLow(26);
      XWeekHighLow(39);
      YearHighLow;
      TwoYearHighLow;
      LifetimeHighLow
    End;  { TProcessData.ProcessSymbolMisc }

  Procedure TProcessData.ProcessSymbolData1Day(Symbol : String;
                                               Bars : TBarList;
                                               Out AverageDailyVolume : Integer);
    Procedure PreviousDay;
      Begin  { PreviousDay }
        If Length(Bars) >= 1 Then
          Begin
            With Bars[Pred(Length(Bars))] Do
              Begin
                FundamentalData.Add('Previous Volume', Symbol, Format('%d',[Volume]));
                FundamentalData.Add('Previous Open', Symbol, Format('%g', [Open]));
                // The older (Windows / Delphi / Realtick) server would read
                // these from the data feed, so we stored it in the F_ file.
                // The newer (Linux / C++ / Proxy) server reads it from the TC_
                // file.  For now we put it in both places.
                FundamentalData.Add('Previous High', Symbol, Format('%g', [High]));
                FundamentalData.Add('Previous Low', Symbol, Format('%g', [Low]));
                TCData.Add('Previous High', Symbol, Format('%g', [High]));
                TCData.Add('Previous Low', Symbol, Format('%g', [Low]))
              End;
            If Length(Bars) >= 2 Then
              FundamentalData.Add('Previous Close', Symbol, Format('%g', [Bars[Length(Bars)-2].Close]))
          End
      End;  { PreviousDay }
    Procedure Volatility(Days : Integer);
      Var
        Value : Double;
      Begin  { Volatility }
        Value := CommonVolatility(Bars, Days);
        If Value > 0 Then
          AdditionalData.Add('Volatility ' + IntToStr(Days) + ' days', Symbol, Format('%g', [Value]))
      End;  { Volatility }
    Procedure BrightVolatility;
      Var
        Numerator : Double;
        Denominator : Integer;
      Procedure AddOne(Months : Integer; Weight : Integer = 1);
        Var
          Value : Double;
        Begin  { AddOne }
          Value := CommonVolatilityMonths(Bars, Months);
          If Value > 0 Then
            Begin
              Numerator := Numerator + Value * Weight;
              Denominator := Denominator + Weight
            End
        End;  { AddOne }
      Begin  { BrightVolatility }
        Numerator := 0.0;
        Denominator := 0;
        AddOne(12);
        AddOne(6);
        AddOne(3);
        AddOne(1, 2);        
        // The original formula come from Don Bright.  But a lot of symbols
        // did not have enough data.  So I modified it to use as much as we
        // have.  The modifications all came from my head.
        If Denominator > 0 Then
          Begin
            //AdditionalData.Add('Volatility 12 months', Symbol, Format('%g', [Value12]));
            //AdditionalData.Add('Volatility 6 months', Symbol, Format('%g', [Value6]));
            //AdditionalData.Add('Volatility 3 months', Symbol, Format('%g', [Value3]));
            //AdditionalData.Add('Volatility 1 months', Symbol, Format('%g', [Value1]));
            Try
              OutputData.Add('Bright Volatility', Symbol,  Format('%g', [Numerator / Denominator / 19.1]))
            Except
            End
          End;
      End;  { BrightVolatility }
    Procedure RangeContraction;
      Function Range(DaysFromEnd : Integer) : Double;
        Begin  { Range }
          With Bars[Length(Bars) - DaysFromEnd - 1] Do
            Result := High - Low
        End;  { Range }
      Var
        I : Integer;
      Begin  { RangeContraction }
        If Length(Bars) > 2 Then
          Begin
            If Range(1) > Range(0) Then
              Begin
                { Range is contracting }
                For I := 1 To Length(Bars) - 2 Do
                  If Range(Succ(I)) <= Range(I) Then
                    { Report when we see the condition change.  If we see the
                      range contract for three days, then we run out of data, we
                      don't know what the value is.  So we report nothing. }
                    Begin
                      TCData.Add('Range Contraction', Symbol, IntToStr(I));
                      Exit
                    End
              End
            Else If Range(1) < Range(0) Then
              { Range is expanding }
              For I := 1 To Length(Bars) - 2 Do
                If Range(Succ(I)) >= Range(I) Then
                  Begin
                    TCData.Add('Range Contraction', Symbol, IntToStr(-I));
                    Exit
                  End
          End
      End;  { RangeContraction }
    Procedure BollingerStdDev(Days : Integer);
      Var
        I, Offset : Integer;
        Closes : Array Of Double;
        Value : Double;
      Begin  { BollingerStdDev }
        Offset := Length(Bars) - Days;
        If Offset >= 0 Then
          Try
            SetLength(Closes, Days);
            For I := 0 To Pred(Days) Do
              Closes[I] := Bars[I + Offset].Close;
            Value := StdDev(Closes);
            If Value > 0 Then
              TCData.Add(Format('%d Day StdDev', [Days]), Symbol, Format('%g', [Value]));
          Except
          End
      End;  { BollingerStdDev }
    Procedure SMA(Days : Integer);
      Var
        I : Integer;
        Total : Double;
      Begin  { SMA }
        If Days <= Length(Bars) Then
          Begin
            Total := 0;
            For I := Length(Bars) - Days To Pred(Length(Bars)) Do
              Total := Total + Bars[I].Close;
            TCData.Add(Format('%d Day SMA', [Days]), Symbol, Format('%g', [Total/Days]));
          End
      End;  { SMA }
    Procedure HighsAndLows;
      Var
        I : Integer;
        Highest, Lowest : Double;
        HighList, LowList : TStringList;
      Procedure NewHigh;
        Begin  { NewHigh }
          Highest := Bars[I].High;
          HighList.Add(Format('%g', [Bars[I].High]));
          HighList.Add(Format('%g', [Bars[I].StartTime]))
        End;  { NewHigh }
      Procedure NewLow;
        Begin  { NewLow }
          Lowest := Bars[I].Low;
          LowList.Add(Format('%g', [Bars[I].Low]));
          LowList.Add(Format('%g', [Bars[I].StartTime]))
        End;  { NewLow }
      Begin  { HighsAndLows }
        If Length(Bars) > 0 Then
          Begin
            TCData.Add('EarliestDate', Symbol, Format('%g', [Bars[Low(Bars)].StartTime]));
            HighList := TStringList.Create;
            LowList := TStringList.Create;
            Try
              Highest := -MaxDouble;
              Lowest := MaxDouble;
              For I := Pred(Length(Bars)) DownTo 0 Do
                Begin
                  If Bars[I].Low < Lowest Then
                    NewLow;
                  If Bars[I].High > Highest Then
                    NewHigh
                End;
              TCData.Add('Highs', Symbol, HighList.CommaText);
              TCData.Add('Lows', Symbol, LowList.CommaText);
            Finally
              HighList.Free;
              LowList.Free
            End
          End
      End;  { HighsAndLows }
    Procedure LastPrice;
      Begin  { LastPrice }
        If Length(Bars) > 0 Then
            OutputData.Add('Last Price', Symbol, Format('%g', [Bars[High(Bars)].Close]))
      End;  { LastPrice }
    Procedure AverageVol(Const Name : String; DayCount : Integer);
      Var
        First, Last, I : Integer;
        TotalVolume : Int64;
      Begin  { AverageVol }
        // Compute it ourselves.
        Last := High(Bars);
        First := Max(0, Length(Bars) - DayCount);
        If First <= Last Then
          Begin
            TotalVolume := 0;
            For I := First To Last Do
              Inc(TotalVolume, Bars[I].Volume);
            AverageDailyVolume := Round(TotalVolume / Succ(Last - First));
            OutputData.Add(Name, Symbol, Format('%d', [AverageDailyVolume]))
          End
      End;  { AverageVol }
    Function Trend(TrendType : TTrendType; Offset : Integer) : Integer;
      Var
        PriceIncrease : Double;
        Count, I, MinDay : Integer;
        PreviousDirection, CurrentDirection : (dNone, dDown, dSame, dUp);
      Begin  { Trend }
        PreviousDirection := dNone;
        Count := 0;
        PriceIncrease := 0;
        If (TrendType = tGreenBar) Then
          MinDay := 0
        Else
          MinDay := 1;
        For I := Pred(Length(Bars)) - Offset DownTo MinDay Do
          Begin
            Case TrendType of
              tOpen : PriceIncrease := Bars[I].Open - Bars[Pred(I)].Open;
              tClose : PriceIncrease := Bars[I].Close - Bars[Pred(I)].Close;
              tHigh : PriceIncrease := Bars[I].High - Bars[Pred(I)].High;
              tLow : PriceIncrease := Bars[I].Low - Bars[Pred(I)].Low;
              tGreenBar : PriceIncrease := Bars[I].Close - Bars[I].Open
            Else Break
            End;
            If PriceIncrease > 0 Then
              CurrentDirection := dUp
            Else If PriceIncrease < 0 Then
              CurrentDirection := dDown
            Else
              CurrentDirection := dSame;
            If CurrentDirection = dSame Then
              Break
            Else If (PreviousDirection <> dNone)
                    And (PreviousDirection <> CurrentDirection) Then
              Break;
            If CurrentDirection = dUp Then
              Inc(Count)
            Else
              Dec(Count);
            PreviousDirection := CurrentDirection;
          End;
        Result := Count;
      End;  { Trend }
    Procedure UpDays;
      Begin  { UpDays }
        TCData.Add('Up Days', Symbol, IntToStr(Trend(tClose, 0)))
      End;  { UpDays }
    Procedure Correlation;
      Var
        Success : Boolean;
        CorrelationSymbol : String;
        M, R2 : Double;
      Begin  { Correlation }
        If Not SymbolIsFuture(Symbol) Then
          Begin
            FindBestCorrelation(Symbol, Bars, Success, CorrelationSymbol, M, R2);
            If Success And (R2 >= MinAcceptableR2) Then
              Begin
                OutputData.Add('Correlation Symbol', Symbol, CorrelationSymbol);
                OutputData.Add('Correlation M', Symbol, Format('%g', [M]));
                FundamentalData.Add('Correlation R2', Symbol, Format('%g', [R2]))
              End;
            DoCloseCloseCorrelation(FuturesDailyData, Bars, Success, M, R2);
            If Success And (R2 >= MinAcceptableR2) Then
              Begin
                OutputData.Add('F Correlation M', Symbol, Format('%g', [M]));
                FundamentalData.Add('F Correlation R2', Symbol, Format('%g', [R2]))
              End
          End
      End;  { Correlation }
    Procedure Consolidation;
      Const
        ChartWidth = 40;
      Var
        ChartTop, ChartBottom : Double;  // Complete chart size, for comparison.
        Top, Bottom, PotentialTop, PotentialBottom : Double;  // The consolidation pattern.
        MaxSize : Double;  // The largest legal consolidation pattern.
        I : Integer;
      Begin  { Consolidation }
        If Length(Bars) >= ChartWidth Then
          Begin
            ChartTop := -MaxDouble;
            ChartBottom := MaxDouble;
            For I := Pred(Length(Bars)) DownTo Length(Bars) - ChartWidth Do
              Begin
                ChartTop := Max(ChartTop, Bars[I].High);
                ChartBottom := Min(ChartBottom, Bars[I].Low)
              End;
            MaxSize := (ChartTop - ChartBottom) * 0.09;
            I := 1;
            Top := -MaxDouble;
            Bottom := MaxDouble;
            Repeat
              If I > ChartWidth Then
                Break;
              With Bars[Length(Bars) - I] Do
                Begin
                  PotentialTop := Max(Top, Max(Open, Close));
                  PotentialBottom := Min(Bottom, Min(Open, Close))
                End;
              If PotentialTop - PotentialBottom > MaxSize Then
                Break;
              Top := PotentialTop;
              Bottom := PotentialBottom;
              Inc(I)
            Until False;
            FundamentalData.Add('Consolidation Days', Symbol, IntToStr(Pred(I)));
            If I > 1 Then
              Begin
                FundamentalData.Add('Consolidation Top', Symbol, Format('%g', [Top]));
                FundamentalData.Add('Consolidation Bottom', Symbol, Format('%g', [Bottom]))
              End
          End
      End;  { Consolidation }
    Procedure AverageTrueRange(Days : Integer; Name : String);
      Var
        I : Integer;
        Total : Double;
        High, Low, PreviousClose, TrueRange : Double;
      Begin  { AverageTrueRange }
        // This minimum value of 3 was recommended by Adam Grimes of SMB.  We
        // use no more bars than requested.  But we'll use as few as 3 if
        // that's all we have.  Otherwise we'd be unable to do anything with
        // a lot of recent IPOs.
        If Length(Bars) >= 3 Then
          Begin
            Days := Min(Days, Pred(Length(Bars)));
            Total := 0.0;
            For I := Length(Bars) - Days To Pred(Length(Bars)) Do
              Begin
                High := Bars[I].High;
                Low := Bars[I].Low;
                PreviousClose := Bars[Pred(I)].Close;
                TrueRange := Max(High, PreviousClose) - Min(Low, PreviousClose);
                Total := Total + TrueRange
              End;
            FundamentalData.Add(Name, Symbol, Format('%g', [Total / Days]))
          End
      End;  { AverageTrueRange }
    Procedure NDayRange(Days : Integer);
      Var
        I : Integer;
        DaysString : String;
        Highest, Lowest : Double;
      Begin  { NDayRange }
        DaysString := IntToStr(Days) + ' Day ';
        If Length(Bars) > Days Then
          TCData.Add(DaysString + 'Close', Symbol, Format('%g', [Bars[Pred(Length(Bars) - Days)].Close]))
        Else If Length(Bars) > 0 Then
          TCData.Add(DaysString + 'Close', Symbol, Format('%g', [Bars[0].Close]));
        If Length(Bars) >= Days Then
          Begin
            Highest := -MaxDouble;
            Lowest := MaxDouble;
            For I := Length(Bars) - Days To Pred(Length(Bars)) Do
              Begin
                Highest := Max(Highest, Bars[I].High);
                Lowest := Min(Lowest, Bars[I].Low)
              End;
            TCData.Add(DaysString + 'High', Symbol, Format('%g', [Highest]));
            TCData.Add(DaysString + 'Low', Symbol, Format('%g', [Lowest]))
          End
      End;  { NDayRange }
    Procedure YearlyClose;
      Var
        EarliestDate : TDateTime;
        I : Integer;
      Begin  { YearlyClose }
        // The 52 week range is more complicated to compute, and it is done
        // elsewhere.
        If Length(Bars) > 0 Then
          Begin
            With Bars[Pred(Length(Bars))] Do
              EarliestDate := IncDay(IncYear(StartTime, -1));
            For I := 0 To Pred(Length(Bars)) Do
              With Bars[I] Do
                If StartTime >= EarliestDate Then
                  Begin
                    TCData.Add('Y Close', Symbol, Format('%g', [Close]));
                    Break
                  End
          End
      End;  { YearlyClose }
    Procedure LastYearClose;
      Var
        FirstDayOfYear : TDateTime;
        I : Integer;
        Year, Unused: Word;
      Begin  { LastYearClose }
        // The naming is getting a little confusing.  This is the last price
        // of the previous calendar year, as opposed to YearlyClose which
        // is the price from a year before the current date.
        If Length(Bars) > 0 Then
          Begin
            { Create date for Jan 1 of the year in which we are adding an alerts_daily.
              After noon we assume we are working on tomorrow's data.  Before noon we
              assume we're working on today's data. }
            FirstDayOfYear := Round(Now);
            Case DayOfTheWeek(FirstDayOfYear) Of
              6 : FirstDayOfYear := FirstDayOfYear + 2;  { Saturday becomes next Monday }
              7 : FirstDayOfYear := FirstDayOfYear + 1;  { Sunday becomes next Monday }
            End;
            DecodeDateTime(FirstDayOfYear, Year, Unused, Unused, Unused, Unused, Unused, Unused);

            FirstDayOfYear := EncodeDateTime(Year, 1, 1, 0, 0, 0, 0);
            For I := Pred(Length(Bars)) DownTo 0 Do
              With Bars[I] Do
                Begin
                  TCData.Add('Last Year Close', Symbol, Format('%g', [Close]));
                  If StartTime < FirstDayOfYear Then
                    Break
                End
          End
      End;  { LastYearClose }
    Procedure RSI(Days : Integer);
      Var
        I : Integer;
        UpAverage : Double;
        DownAverage : Double;
        Change : Double;
      Begin  { RSI }
        //needs x+1 days to compute x price changes
        If Days + 1 <= Length(Bars) Then
          Begin
            UpAverage := 0;
            DownAverage := 0;
            //initial smoothed value
            For I := 1 To Days Do
              Begin
                Change := Bars[I].Close - Bars[Pred(I)].Close;
                If Change < 0 Then
                  DownAverage := DownAverage + abs(Change)
                Else
                  UpAverage := UpAverage + Change
              End;
            UpAverage := UpAverage / Days;
            DownAverage := DownAverage / Days;
            //subsequent updates
            For I := Succ(Days) To Pred(Length(Bars)) Do
              Begin
                Change := Bars[I].Close - Bars[Pred(I)].Close;
                If Change < 0 Then
                  Begin
                    DownAverage :=  (DownAverage * (Days - 1) + abs(Change))/Days;
                    UpAverage := (UpAverage * (Days - 1))/Days
                  End
                Else
                  Begin
                    UpAverage :=  (UpAverage * (Days - 1) + Change)/Days;
                    DownAverage := (DownAverage * (Days - 1))/Days
                  End;
              End;
            If DownAverage = 0 Then
              FundamentalData.Add(Format('%d Day RSI', [Days]), Symbol, Format('%g', [100.00]))
            Else If Not (UpAverage = 0) And Not (DownAverage = 0) Then
              FundamentalData.Add(Format('%d Day RSI', [Days]), Symbol, Format('%g', [(100 - 100/(1 + (UpAverage/DownAverage)))]))
          End
      End;  { RSI }
    Procedure PristineLists();
      Var
       I, LastBar, HighUpDays, PrevHighUpDays, GreenBars, PrevGreenBars : Integer;
       Range, PrevRange, AvgRange, BodySize, TailSize, SMA : Double;
       Tail : (TailBottoming, TailNone, TailTopping);
       ChangingOfTheGuard : (COGMinus, COGNone, COGPlus);
       RangeBar : (RangeBarNarrow, RangeBarWide, RangeBarNone);
       CloseTotal, RangeTotal,  Price, ClosePositionInRange, OpenPositionInRange : Double;
      Begin  { PristineLists }
        If Length(Bars) >= 3 Then
          Begin
            HighUpDays := Trend(tHigh, 0);
            PrevHighUpDays := Trend(tHigh, 1);
            GreenBars := Trend(tGreenBar,0);
            PrevGreenBars := Trend(tGreenBar, 1);
            LastBar := Pred(Length(Bars));
            Range := Bars[LastBar].High - Bars[LastBar].Low;
            PrevRange := Bars[LastBar-1].High - Bars[LastBar-1].Low;
            BodySize := Abs(Bars[LastBar].Close - Bars[LastBar].Open);
            Price:= Bars[LastBar].Close;

            {Changing of the Guard}
            If  (GreenBars = 1) And (PrevGreenBars <= -3) Then
              ChangingOfTheGuard := COGPlus
            Else If (GreenBars = -1) And (PrevGreenBars >= 3) Then
              ChangingOfTheGuard := COGMinus
            Else
              ChangingOfTheGuard := COGNone;

            { Topping/Bottoming Tail }
            Tail := TailNone;
            TailSize := 0;
            { Figure out if we have enough bars in the proper direction for Topping or Bottoming
              and calculate tail size }
            If (PrevGreenBars >= 3) Then
              Begin
                Tail := TailTopping;
                TailSize := Bars[LastBar].High - Max(Bars[LastBar].Open, Bars[LastBar].Close);
              End
            Else If (PrevGreenBars <= -3) Then
              Begin
                Tail := TailBottoming;
                TailSize := Min(Bars[LastBar].Open, Bars[LastBar].Close) - Bars[LastBar].Low;
              End;
            { If we have the proper direction, make sure that the tail is large enough to qualify }
            If (Tail <> TailNone) And
                (Not((Range >= PrevRange * 0.5) And
                                           (TailSize >= BodySize * 0.6) And
                                           (TailSize >= Range * 0.3333333))) Then
                  Tail := TailNone;

            { Pristine Buy/Sell Setups }
            If (HighUpDays <= -3) Or (GreenBars <= -3) Then
                PristineData.Add('PBS', Symbol, IntToStr(1))
            Else If (HighUpDays >= 3) Or (GreenBars >= 3) Then
                PristineData.Add('PSS', Symbol, IntToStr(1));

            { Pristine Buy/Sell Setups+ }
            If (ChangingOfTheGuard = COGPlus)
              Or (Tail = TailBottoming) Then
              PristineData.Add('PBS+', Symbol, IntToStr(1))
            Else If (ChangingOfTheGuard = COGMinus)
              Or (Tail = TailTopping) Then
              PristineData.Add('PSS+', Symbol, IntToStr(1));

            { For the rest we need 20 days of previous history for SMA, and Average Range }
            If 21 <= Length(Bars) Then
              Begin
                CloseTotal := 0;
                RangeTotal := 0;
                For I := Length(Bars) - 21 To Length(Bars) - 2 Do
                  Begin
                    CloseTotal := CloseTotal + Bars[I].Close;
                    RangeTotal := RangeTotal + Bars[I].High - Bars[I].Low
                  End;
                SMA := CloseTotal/20;
                AvgRange := RangeTotal/20;

                { Wide Range Bar, this is both its own list, and a criterion for some other lists }
                If Range <= 0.65 * AvgRange Then
                  RangeBar := RangeBarNarrow   
                Else If Range >= 1.7 * AvgRange Then
                  Begin
                    RangeBar := RangeBarWide;
                    PristineData.Add('WRB', Symbol, IntToStr(1))
                  End
                Else
                  RangeBar := RangeBarNone;
                // Climatic Buy/Sell
                If ((PrevGreenBars <= -3) Or (PrevHighUpDays <= -3)) And (Price <= 0.92 * SMA)
                  And ((Tail = TailBottoming) Or (RangeBar = RangeBarNarrow) Or (ChangingOfTheGuard = COGPlus))
                Then
                  PristineData.Add('CBS', Symbol, IntToStr(1))
                Else If ((PrevGreenBars >= 3) Or (PrevHighUpDays >= 3)) And (Price >= 1.08 * SMA)
                  And ((Tail = TailTopping) Or (RangeBar = RangeBarNarrow) Or (ChangingOfTheGuard = COGMinus))
                Then
                  PristineData.Add('CSS', Symbol, IntToStr(1));

                { Daily Bullish/Bearish 20/20 }
                If Range > 0.0 Then
                  Begin
                    ClosePositionInRange := (Bars[LastBar].Close - Bars[LastBar].Low)/Range;
                    OpenPositionInRange := (Bars[LastBar].Open - Bars[LastBar].Low)/Range;
                    If (OpenPositionInRange <= 0.20) And (ClosePositionInRange >= 0.80)
                      And (PrevGreenBars >= 1)And (RangeBar = RangeBarWide) Then
                        PristineData.Add('D20+', Symbol, IntToStr(1))
                    Else If (ClosePositionInRange <= 0.20) And (OpenPositionInRange >= 0.80)
                      And (PrevGreenBars <= -1) And (RangeBar = RangeBarWide) Then
                        PristineData.Add('D20-', Symbol, IntToStr(1))
                  End
              End
          End
      End;  { PristineLists }
    Procedure ComputeADX(N : Integer = 14);
      Type
        TDoubleArray = Array Of Double;
      Function MovingEMA(Const Source : TDoubleArray) : TDoubleArray;
        Var
          I : Integer;
          OldPart, NewPart, Previous : Double;
        Begin  { MovingEMA }
          SetLength(Result, Length(Source));
          Previous := NaN;
          MovingEMA[0] := Source[0];
          // This is different from the formula suggested by http://fxtrade.oanda.com/learn/graphs/indicators/adx
          // This is a more common formula which gives us results closer to what realtick sees.
          //NewPart := 2.0 / Succ(N);
          // This version is popular on a lot of web sites, but not all.  This
          // appears to match eSignal a lot more closely.  (It's still not a
          // perfect match.  It seems that eSignal starts with an SMA of the
          // first N-1 bars, or something like that.)
          NewPart := 1.0 / N;
          OldPart := 1.0 - NewPart;
          For I := Low(Source) To High(Source) Do
            Begin
              // Sometimes the first item was a NAN, so the result of the entire
              // EMA was a NAN.  Presumably other items, besides the first,
              // could be NAN.  We just ignore the NANs.
              If IsNan(Previous) Then
                Previous := Source[I]
              Else If Not IsNan(Source[I]) Then
                Previous := OldPart * Previous + NewPart * Source[I];
              // Else Previous := Previous
              MovingEma[I] := Previous
            End
        End;  { MovingEMA }
      Function TrueRange(Const Yesterday, Today : TBarData) : Double;
        Begin  { TrueRange }
          If Yesterday.Close > Today.High Then
            TrueRange := Yesterday.Close - Today.Low
          Else If Yesterday.Close < Today.Low Then
            TrueRange := Today.High - Yesterday.Close
          Else
            TrueRange := Today.High - Yesterday.Low
        End;  { TrueRange }
      Var
        A, B : Double;  // Possible values for +DM and -DM.
        PDM, MDM,  // +DM, -DM
        TR,  // True Range.
        EmaPDM, EmaMDM, ATR,  // EMA of +DM, EMA of -DM, EMA of TR.
        DX, ADX : TDoubleArray;
        PDI, MDI : Double;  // +DI, -DI
      Procedure DebugDump;
        Var
          I : Integer;
        Begin  { DebugDump }
          I := Length(PDM);
          Assert(I = Length(MDM));
          Assert(I = Length(TR));
          Assert(I = Length(EmaPDM));
          Assert(I = Length(EmaMDM));
          Assert(I = Length(ATR));
          Assert(I = Length(DX));
          Assert(I = Length(ADX));
          MultiWriteLn('NAN detected in ADX of ' + Symbol);
          MultiWriteLn('PDM,MDM,TR,EmaPDM,EmaMDM,ATR,DX,ADX');
          For I := Low(PDM) To High(PDM) Do
            MultiWriteLn(Format('%g,%g,%g,%g,%g,%g,%g,%g', [PDM[I],MDM[I],TR[I],EmaPDM[I],EmaMDM[I],ATR[I],DX[I],ADX[I]]))
        End;  { DebugDump }
      Var
        I : Integer;
      Begin  { ComputeADX }
        // http://fxtrade.oanda.com/learn/graphs/indicators/adx

        // Stupid compiler warnings.
        PDI := 0;
        MDI := 0;
        SetLength(ATR, 0);
        SetLength(EmaPDM, 0);
        SetLength(EmaMDM, 0);
        SetLength(ADX, 0);

        If Length(Bars) > Succ(N) Then
          Try
            SetLength(PDM, Pred(Length(Bars)));
            SetLength(MDM, Pred(Length(Bars)));
            SetLength(TR, Pred(Length(Bars)));
            For I := 1 To Pred(Length(Bars)) Do
              Begin
                A := Max(Bars[I].High - Bars[Pred(I)].High, 0.0);
                B := Max(Bars[Pred(I)].Low - Bars[I].Low, 0.0);
                If (A > B) Then
                  Begin
                    PDM[Pred(I)] := A;
                    MDM[Pred(I)] := 0.0
                  End
                Else
                  Begin
                    PDM[Pred(I)] := 0.0;
                    MDM[Pred(I)] := B
                  End;
                TR[Pred(I)] := TrueRange(Bars[Pred(I)], Bars[I])
              End;
            EmaPDM := MovingEMA(PDM);
            EmaMDM := MovingEMA(MDM);
            ATR := MovingEMA(TR);
            SetLength(DX, Length(ATR));
            For I := Low(DX) To High(DX) Do
              Begin
                If ATR[I] > 0 Then
                  Begin
                    PDI := EmaPDM[I] / ATR[I];
                    MDI := EmaMDM[I] / ATR[I]
                  End
                Else
                  Begin
                    // Presumably the ATR could be 0.  In that case we might
                    // throw an exception and report nothing for this stock.  In
                    // fact, we see that some.  Presumably the ATR would be 0
                    // for the first few entries, but would not always be 0.  In
                    // those cases, if we are careful, we could avoid the
                    // exception and print something reasonable.
                    PDI := NaN;
                    MDI := NaN
                  End;
                If (PDI + MDI) > 0 Then
                  // Strange.  It seems that this next line will produce a NAN,
                  // not an exception, when PDI and MDI are both 0.  So this
                  // if statement is redundant.  That doesn't make sense.
                  // On further analysis, it seems that certain computers will
                  // signal an exception, and others will return NaN.  There's
                  // no obvious reason, like a different version of the OS.
                  DX[I] := Abs(PDI - MDI) / (PDI + MDI)
                Else
                  DX[I] := 0
              End;
            ADX := MovingEMA(DX);
            FundamentalData.Add(Format('ADX %d', [N]),
                                Symbol, Format('%g', [ADX[Pred(Length(ADX))]*100.0]));
            If IsNan(ADX[Pred(Length(ADX))]*100.0) Then
              DebugDump;
            FundamentalData.Add(Format('+DI %d', [N]),
                                Symbol, Format('%g', [PDI*100.0]));
            FundamentalData.Add(Format('-DI %d', [N]),
                                Symbol, Format('%g', [MDI*100.0]))
          Except
            // Presumably we could get a /0 error.
          End
      End;  { ComputeADX }
    Begin  { TProcessData.ProcessSymbolData1Day }
      AverageDailyVolume := 0;  // Set a good default value, just in case.
      If Length(Bars) > 0 Then
        Begin
          Assert(Bars[0].StartTime <= Bars[Pred(Length(Bars))].StartTime);
          SMA(200);
          SMA(50);
          SMA(20);
          SMA(10);
          SMA(8);
          SMA(4);
          HighsAndLows;
          LastPrice;
          AverageVol('Avg Daily Volume', 10);
          AverageVol('Avg Daily Volume 5', 5);
          AverageVol('Avg Daily Volume 3M', 63);  // 3 Months
          AverageVol('Avg Daily Volume 60', 60);  // For SMB.
          UpDays;
          Correlation;
          RangeContraction;
          //Volatility(9);
          //Volatility(14);
          //Volatility(20);
          //Volatility(50);
          //Volatility(100);
          BrightVolatility;
          PreviousDay;
          BollingerStdDev(20);
          BollingerStdDev(10);
          Consolidation;
          AverageTrueRange(14, 'Average True Range');
          // SMB asked for an ATR for one quarter.  The idea was to avoid
          // changes around the time that dividends are paid.  According to
          // http://wiki.answers.com/Q/What_is_the_average_number_of_trading_days_in_a_year_on_the_stock_exchange
          // there are 252 trading days in an average year, so that makes 63
          // days in a quarter.
          AverageTrueRange(63, 'ATR Q');
          // This is also for SMB.  63 days is used for the intraday formulas.
          // 20 is used when we build the universe each night.  That is
          // primarily to be consistent with the previous code.
          AverageTrueRange(20, 'ATR 20');
          NDayRange(5);
          NDayRange(10);
          NDayRange(20);
          RSI(14);
          YearlyClose;
          PristineLists;
          ComputeADX;
          LastYearClose
        End
    End;  { TProcessData.ProcessSymbolData1Day }

  // Standard Deviation = Sqrt((sum(x^2 - (Sum(X)^2 / n)) / (N - 1))

  Constructor TProcessData.Create;
    Begin
      StandardCandlesInput := TTwoDArray.Create
    End;
  Procedure TProcessData.InitializeDumpCandles;
    Begin
      FDumpCandles := TDumpCandles.Create(Round(MarketStart * 60), Round(MarketStart * 60 + Periods * 15))
    End;

  Procedure TProcessData.SetOutputFiles(BaseName : String);
    Var
      Path, Remainder : String;
    Begin
      FlushOutputFiles;
      Path := ExtractFilePath(BaseName);
      Remainder := ExtractFileName(BaseName);

      { This was the original file, hence the simple name.  This contains general
        background data which could be useful at different times. }
      OutputData := TTwoDArrayWriter.Create(BaseName,
        ['Volume Break', 'Tick Volatility', 'Bright Volatility', 'Last Price',
         'Avg Daily Volume', 'Avg Daily Volume 5', 'Avg Daily Volume 60', 'Avg Daily Volume 3M', 'Correlation Symbol', 'Correlation M',
         'F Correlation M', 'Shares per Print', 'Listed Exchange']);

      { This is the average volume for each time period.  This was seperated
        from OutputData primarily to make both files more readable.  Most column
        names are only numbers, similar to the StandardCandles file.  The file
        name is required to disambiguate. }
      VolumeData := TTwoDArrayWriter.Create(Path + 'V_' + Remainder);

      { This is similar to OutputData, except that it should expire at the end
        of the day.  Ideally, if the overnight data program crashed, this file
        would be deleted, but the old values from OutputData would be used.  We
        don't do that, but the possibility exists.  TC stands for "time
        critical." }
      TCData := TTwoDArrayWriter.Create(Path + 'TC_' + Remainder,
       ['Range Contraction', '8 Day SMA', '20 Day SMA', '50 Day SMA',
        '200 Day SMA', '20 Day StdDev', 'EarliestDate', 'Highs', 'Lows',
        'Up Days', '5 Day Close', '5 Day High', '5 Day Low', '10 Day Close',
        '10 Day High', '10 Day Low','20 Day Close','20 Day High','20 Day Low',
        'Y Close', '10 Day SMA', '10 Day StdDev', '4 Day SMA',
        'Last Year Close']);

      { This stores historical volume blocks.  These should look similar to what
        it would look like if we were receiving the tick data in real time.  We
        approximate this using 1 minute candles.  We massage that data to put it
        into the same format as the realtime data, although we're always loosing
        something.  The row names are different in here than in most of the files.
        The name of the row includes the symbol and the number of shares per
        block.  Currently we only support one number of shares per block, so
        this don't do much.  However, many parts of the code were written to
        support other options, so this is consistant. }
      VolumeBlocks := TTwoDArrayWriter.Create(Path + 'VB_' + Remainder, ['Vol']);

      { This takes the candlestick data and stores it in a local file with very
        little masaging.  The row is the symbol name, and the column name is the
        number of minutes per candle.  (VolumeBlocks should have been organized
        this way.) }
      StandardCandles := TTwoDArrayWriter.Create(Path + 'SC_' + Remainder);

      { This is data which is used only by the database, not the Delphi software
        which generates the alerts.  This was seperated out to make sure that
        software doesn't waste memory on things it doesn't need.  Some of the
        files above are used only by the alerts generation software, and some
        are used by both that software and the database.  The name "Fundamental"
        comes from the history of this file.  It is no longer acurate. }
      FundamentalData := TTwoDArrayWriter.Create(Path + 'F_' + Remainder,
        ['Company Name', 'Listed Exchange', 'Lifetime High', 'Lifetime Low',
         '52 Week High', '52 Week Low', 'Previous High', 'Previous Low',
         'Previous Close', 'Previous Open', 'Correlation R2', 'F Correlation R2',
         'Consolidation Days', 'Consolidation Top', 'Consolidation Bottom',
         'Average True Range', 'ATR 20', 'ATR Q', 'Average Bar Size 5', 'Bunny 130', 'Previous Volume', 'Prev Put Volume',
         'Prev Call Volume', 'Dividend', 'Earnings', 'P/E Ratio', 'Beta',
         'Yield', 'Market Cap', 'Shares Outstanding', 'EPS Net Income',
         'Income', 'Assets', 'Debt', '14 Day RSI',
         'ADX 14', '+DI 14', '-DI 14', 'Short Interest', 'NAICS Code', 'NAICS',
         'EnterpriseValue', 'Revenue', 'ShortGrowth', 'QuarterlyRevenueGrowth',
         'QuarterlyEarningsGrowth', 'PercentHeldByInsiders',
         'PercentHeldByInstitutions', 'ShortRatio', 'ShortPercentOfFloat', 'Cash',
         'EstimatedQuarterlyEarningsGrowth', 'EstimatedAnnualEarningsGrowth',
         'PEGRatio', 'CUSIP', 'Forward Dividend Rate', 'Trailing Dividend Rate',
         'Float', 'Profile', '13 Week High', '13 Week Low', '26 Week High',
         '26 Week Low', '39 Week High', '39 Week Low', '104 Week High',
         '104 Week Low', '2 Day AVWAP PV', '2 Day AVWAP TV', '3 Day AVWAP PV',
         '3 Day AVWAP TV', '4 Day AVWAP PV', '4 Day AVWAP TV', '5 Day AVWAP PV',
         '5 Day AVWAP TV', 'DaysToCover' ]);

      { This file stores a 1 for each stock if it qualifies as that particular
        Pristine strategy.  This data will be processed by another program which
        will pull out a list of stocks for each category and add them to the
        database under a specified account (currently Pristine_Dynamic_Test) }
      PristineData := TTwoDArrayWriter.Create(Path + 'Pristine_' + Remainder,
      ['PBS', 'PSS', 'PBS+', 'PSS+', 'WRB', 'CBS', 'CSS', 'D20+', 'D20-']);

      { This file is used for debugging.  You can put things into this file and
        read them with notepad.  They are not used anywhere else. }
      AdditionalData := TTwoDArrayWriter.Create(Path + 'Additional_' + Remainder)
    End;

  Procedure TProcessData.FlushOutputFiles;
    Begin
      OutputData.Free;
      OutputData := Nil;
      VolumeData.Free;
      VolumeData := Nil;
      TCData.Free;
      TCData := Nil;
      VolumeBlocks.Free;
      VolumeBlocks := Nil;
      StandardCandles.Free;
      StandardCandles := Nil;
      FundamentalData.Free;
      FundamentalData := Nil;
      PristineData.Free;
      PristineData := Nil;
      AdditionalData.Free;
      AdditionalData := Nil;
      FDumpCandles.Free;
      FDumpCandles := TDumpCandles.Create(Round(MarketStart * 60), Round(MarketStart * 60 + Periods * 15))
    End;

  Destructor TProcessData.Destroy;
    Begin
      FlushOutputFiles;
      StandardCandlesInput.Free;
      FDumpCandles.Free
    End;

  Procedure TProcessData.ClearAll;
    Begin
      FlushOutputFiles;
      SetLength(PossibleCorrelations, 0)
    End;

  Procedure TProcessData.SetPeriods(P : Integer);
    Begin
      Periods := P
    End;

  Procedure TProcessData.SetStartTime(T : Double);
    Begin
      MarketStart := T
    End;

  Procedure TProcessData.BuildCharts(Symbol : String;
                                     Data1Day : TBarList);
    Var
      Prices : TPrices;
      SourceIndex, DestinationIndex : Integer;
      CurrentWeek, PreviousWeek : Word;
      DateCutoff : Double;
    Begin
      If Length(Data1Day) > 0 Then
        Begin
          { Daily Charts }
          SourceIndex := Pred(Length(Data1Day));
          DateCutoff := Round(Data1Day[SourceIndex].StartTime);
          DateCutoff := IncMonth(DateCutoff, -3);
          Repeat
            If Data1Day[SourceIndex].Close <= 0 Then
              Break;
            Dec(SourceIndex);
            If SourceIndex < 0 Then
              Break;
            If Round(Data1Day[SourceIndex].StartTime) <= DateCutoff Then
              Break
          Until False;
          Inc(SourceIndex);
          If SourceIndex < Length(Data1Day) Then
            Begin
              SetLength(Prices, Length(Data1Day) - SourceIndex);
              For DestinationIndex := 0 To Pred(Length(Prices)) Do
                Begin
                  Prices[DestinationIndex] := Data1Day[SourceIndex].Close;
                  Inc(SourceIndex)
                End;
              CreateAndSaveChart(ChartsDirectory
                                 + '\D_'
                                 + EncodeFileName(Symbol)
                                 + '.gif',
                                 Prices,
                                 66)
            End;
          { Yearly Charts }
          DestinationIndex := 0;
          PreviousWeek := High(PreviousWeek);
          SourceIndex := Pred(Length(Data1Day));
          DateCutoff := Round(Data1Day[SourceIndex].StartTime);
          DateCutoff := IncYear(DateCutoff, -1);
          Repeat
            If Data1Day[SourceIndex].Close <= 0 Then
              Break;
            CurrentWeek := WeekOf(Round(Data1Day[SourceIndex].StartTime));
            If CurrentWeek <> PreviousWeek Then
              Begin
                PreviousWeek := CurrentWeek;
                Inc(DestinationIndex)
              End;
            Dec(SourceIndex);
            If SourceIndex < 0 Then
              Break;
            If Round(Data1Day[SourceIndex].StartTime) <= DateCutoff Then
              Break;
          Until False;
          If DestinationIndex > 0 Then
            Begin
              Inc(SourceIndex);
              SetLength(Prices, DestinationIndex);
              PreviousWeek := High(PreviousWeek);
              DestinationIndex := -1;
              For SourceIndex := SourceIndex To Pred(Length(Data1Day)) Do
                Begin
                  CurrentWeek := WeekOf(Round(Data1Day[SourceIndex].StartTime));
                  If CurrentWeek <> PreviousWeek Then
                    Begin
                      PreviousWeek := CurrentWeek;
                      Inc(DestinationIndex)
                    End;
                  Prices[DestinationIndex] := Data1Day[SourceIndex].Close
                End;
              If (Succ(DestinationIndex) <> Length(Prices)) Then
                MultiWriteLn(Symbol);
              Assert(Succ(DestinationIndex) = Length(Prices));
              CreateAndSaveChart(ChartsDirectory
                                 + '\W_'
                                 + EncodeFileName(Symbol)
                                 + '.gif',
                                 Prices,
                                 53)
            End
        End
    End;

End.