dataframes source code

1 module dataframes;
2 
3 import std.traits;
4 import std.range;
5 import std.algorithm;
6 import std.array;
7 import std.format;
8 import std.json;
9 import std.conv : text, to;
10 import std.variant;
11 import core.exception;
12 import std.datetime;
13 
14 public
15 {
16     import dataframes.helpers;
17     import dataframes.columns;
18     import dataframes.rows;
19 }
20 
21 /**
22  * DataFrame
23  */
24 class DataFrame(T)
25 {
26     alias fieldNames = FieldNameTuple!(T);
27     alias fieldTypes = FieldTypeTuple!(T);
28 
29     size_t[] index;
30     alias RowType = Row!T;
31 
32     static foreach(idx, type; fieldTypes)
33     {
34         mixin(Column!(type).stringof ~ " " ~ fieldNames[idx] ~ ";");
35     }
36 
37     private static string initializerArgs()
38     {
39         import std.array;
40 
41         string[] args;
42         static foreach(idx, type; fieldTypes)
43         {
44             // Ex: double[] price = []
45             args ~= type.stringof ~ "[] " ~ fieldNames[idx] ~ " = []";
46         }
47         return args.join(", ");
48     }
49 
50     private static string initializerContent()
51     {
52         string content;
53         content ~= "size_t dfSize = 0;";
54         content ~= "bool allColumnSameSize = true;";
55 
56         static foreach(idx, name; fieldNames)
57         {
58             // Ex: if (dfSize == 0) dfSize = FIELD.length
59             content ~= "if (dfSize == 0) dfSize = " ~ name ~ ".length;";
60             // Ex: if (dfSize != FIELD.length && FIELD.length > 0)
61             //         allColumnSameSize = false;
62             content ~= "if (dfSize != " ~ name ~ ".length && " ~ name ~ ".length > 0) allColumnSameSize = false;";
63             // Ex: if (FIELD.length > 0)
64             //         this.FIELD = FIELD;
65             //     else
66             //         this.FIELD.values.length = dfSize;
67             content ~= "if (" ~ name ~ ".length > 0) this." ~ name ~ ".values = " ~ name ~ "; else this." ~ name ~ ".values.length = dfSize;";
68         }
69 
70         content ~= "if (!allColumnSameSize) throw new DataFrameException(\"All arrays must be of the same length\");";
71         content ~= "this.index = iota(length).array;";
72 
73         return content;
74     }
75 
76     mixin("this(" ~ initializerArgs ~ "){" ~ initializerContent ~ "}");
77 
78     size_t opDollar(size_t pos)()
79     {
80         return length;
81     }
82 
83     auto opSlice()(size_t start, size_t end)
84     {
85         return iota(start, end).map!(idx => row(idx)).array;
86     }
87 
88     /**
89      * Access the DataFrame column by label
90      */
91     Variant opIndex(string label)
92     {
93         Variant output;
94     s1: switch (label)
95         {
96             static foreach(name; fieldNames)
97             {
98                 mixin("case \"" ~ name ~ "\": output = this." ~ name ~ "; break s1;");
99             }
100         default:
101             throw new RangeError("invalid label");
102         }
103         return output;
104     }
105 
106     /**
107      * Access the DataFrame column by index
108      */
109     Variant opIndex(size_t idx)
110     {
111         return opIndex(columnNames[idx]);
112     }
113 
114     /**
115      * Get the list of column names
116      */
117     string[] columnNames()
118     {
119         string[] names;
120         foreach(name; fieldNames)
121             names ~= name;
122 
123         return names;
124     }
125 
126     /**
127      * Get the number of columns
128      */
129     size_t ncol()
130     {
131         return fieldNames.length;
132     }
133 
134     /**
135      * Get the number of rows
136      */
137     size_t length()
138     {
139         return  __traits(getMember, this, fieldNames[0]).length;
140     }
141 
142     /**
143      * Get the number of rows. Alias of `length`
144      */
145     size_t nrow()
146     {
147         return length;
148     }
149 
150     /**
151      * Add a row to the dataframe.
152      */
153     void add(T data)
154     {
155         // Expands to this.fieldName ~= data.fieldName;
156         static foreach(name; fieldNames)
157             mixin("this." ~ name ~ " ~= " ~ "data." ~ name ~ ";");
158 
159         this.index ~= length - 1;
160     }
161 
162     RowType row(size_t idx)
163     {
164         return Row!T(this, idx);
165     }
166 
167     RowType[] rows()
168     {
169         RowType[] output;
170         output.length = length;
171         foreach(idx; 0..length)
172             output[idx] = RowType(this, idx);
173         return output;
174     }
175 
176     RowType[] head(size_t count = 10)
177     {
178         if (length < count)
179             count = length;
180 
181         RowType[] output;
182         output.length = count;
183         foreach(idx; 0..count)
184             output[idx] = RowType(this, idx);
185 
186         return output;
187     }
188 
189     RowType[] tail(size_t count = 10)
190     {
191         auto start = length - count;
192         if (length < count)
193         {
194             start = 0;
195             count = length;
196         }
197 
198         RowType[] output;
199         output.length = count;
200         foreach(idx; start..length)
201             output[idx-start] = RowType(this, idx);
202 
203         return output;
204     }
205 
206     private string formatRow(size_t idx)
207     {
208         auto output = appender!string;
209 
210         output ~= format("%10s  ", idx);
211 
212         static foreach(name; fieldNames)
213         {
214             static if(isNumeric!(typeof( __traits(getMember, this, name).values[0])))
215             {
216                 if (is(typeof( __traits(getMember, this, name).values[0]) == double))
217                     output.put(format("%10.2f  ", __traits(getMember, this, name)[idx]));
218                 else
219                     output.put(format("%10s  ", __traits(getMember, this, name)[idx]));
220             }
221             else
222                 output.put(format("%-10s  ", __traits(getMember, this, name)[idx]));
223         }
224 
225         return output.data ~ "\n";
226     }
227 
228     private string formatHeader()
229     {
230         auto output = appender!string;
231 
232         output ~= format("%10s  ", "Index");
233 
234         if (length > 0)
235         {
236             foreach(cname; fieldNames)
237             {
238                 static if(isNumeric!(typeof( __traits(getMember, this, cname).values[0])))
239                     output.put(format("%10s  ", cname));
240                 else
241                     output.put(format("%-10s  ", cname));
242             }
243 
244             output ~= "\n";
245         }
246 
247         return output.data;
248     }
249 
250     override string toString()
251     {
252         auto output = appender!string;
253         // TODO: Print Title and field information
254 
255         output ~= formatHeader;
256 
257         if (length <= 10)
258         {
259             foreach(r; this.rows)
260                 output ~= formatRow(r.index);
261         }
262         else
263         {
264             this.head(5).each!(r => output ~= formatRow(r.index));
265 
266             output.put(format("%10s\n%10s\n", ".", "."));
267 
268             this.tail(5).each!(r => output ~= formatRow(r.index));
269         }
270 
271         output.put(i"\n$(length) rows".text);
272 
273         return output.data;
274     }
275 
276     DataFrame!T toDataFrame()
277     {
278         return toDataFrame!T;
279     }
280 
281     DataFrame!T1 toDataFrame(T1)()
282     {
283         auto output = new DataFrame!T1;
284         foreach(r; this.rows)
285             output.add(r.toDataFrameStruct!T1);
286 
287         return output;
288     }
289 
290     Column!T1 toColumn(T1, string colName)(RowType[] rows)
291     {
292         Column!T1 output;
293         foreach(row; rows)
294             mixin("output ~= row." ~ colName ~ ";");
295 
296         return output;
297     }
298 
299     private void applyLogic(T1)(T1[] data, ref T1 field, string logic)
300     {
301         switch(logic)
302         {
303         case "first":
304             if (data.length > 0)
305                 field = data[0];
306             break;
307         case "last":
308             if (data.length > 0)
309                 field = data[$-1];
310             break;
311         case "max":
312             field = data.maxElement;
313             break;
314         case "min":
315             field = data.minElement;
316             break;
317         case "count":
318             static if (!is(T1 == Date))
319                 field = data.length.to!T1;
320             break;
321         case "sum":
322             static if (isNumeric!(T1))
323                 field = data.sum;
324             break;
325         default:
326             break;
327         }
328     }
329 
330     private void setAggregateLogic(RowType[] rowsdata, ref T row, string fieldName, string logic)
331     {
332     ss:switch(fieldName)
333         {
334             static foreach(idx, name; fieldNames)
335             {
336                 // Ex:
337                 // case "<column-name>":
338                 //     auto col = toColumn!(<type>, <name>)(rowsdata);
339                 //     applyLogic!(Type)(col.values, row.<name>, logic);
340                 //     break;
341                 mixin("case name:
342                            auto col = toColumn!(" ~ fieldTypes[idx].stringof ~ ", \"" ~ name ~ "\")(rowsdata);
343                            applyLogic!(" ~ fieldTypes[idx].stringof ~ ")(col.values, row." ~ name ~ ", logic);
344                            break ss;");
345             }
346         default:
347             break;
348         }
349     }
350 
351     DataFrame!T resample(T1)(T1 grouped, string[string] logic)
352     {
353         auto df = new DataFrame!T;
354         foreach(grp; grouped)
355         {
356             T output;
357             foreach(kv; logic.byKeyValue)
358                 setAggregateLogic(grp.array, output, kv.key, kv.value);
359 
360             df.add(output);
361         }
362 
363         return df;
364     }
365 
366     // Resample using same logic for all the fields
367     DataFrame!T resample(T1)(T1 grouped, string logic)
368     {
369         string[string] logic2;
370         static foreach(name; fieldNames)
371             logic2[name] = logic;
372         return resample(grouped, logic2);
373     }
374 
375     // Apply logic in the same order of the struct fields
376     DataFrame!T resample(T1)(T1 grouped, string[] logic)
377     {
378         string[string] logic2;
379         static foreach(idx, name; fieldNames)
380         {
381             if (logic.length > idx)
382                 logic2[name] = logic[idx];
383         }
384         return resample(grouped, logic2);
385     }
386 
387     JSONValue toJSON()
388     {
389         JSONValue output = JSONValue.emptyArray;
390         foreach(row; this.rows)
391             output.array ~= row.toJSON;
392 
393         return output;
394     }
395 }
396 
397 unittest
398 {
399     struct Item
400     {
401         string name;
402         double price;
403         int quantity;
404         double totalPrice;
405     }
406 
407     auto df = new DataFrame!Item(
408         name: ["A", "B", "C", "D"],
409         price: [149.0, 799.0, 399.0, 299.0],
410         quantity: [3, 1, 2, 1]
411         );
412 
413     assert(df.ncol == 4);
414     assert(df.nrow == 4);
415     assert(df.length == 4);
416 
417     assert(df.columnNames == ["name", "price", "quantity", "totalPrice"]);
418 
419     // Try to add column data of different length
420     // than the other columns.
421     import std.exception;
422     assertThrown!DataFrameException(df.totalPrice = [10, 20]);
423 
424     // Values for this column not set during initialization,
425     // but DataFrame should set the size as same as other columns
426     assert(df.totalPrice.length == 4);
427 
428     auto item = Item("E", 49.0, 10);
429     df.add(item);
430     assert(df.nrow == 5);
431 
432     auto row = Row!Item(df, 0);
433     assert(row.name == "A");
434     assert(row.price == 149.0);
435     assert(row.quantity == 3);
436 
437     auto row2 = Row!Item(df, 3);
438     assert(row2.name == "D");
439     assert(row2.price == 299.0);
440     assert(row2.quantity == 1);
441 
442     assert(df.row(0).name == "A");
443 
444     df.totalPrice = df.price * df.quantity;
445     assert(df.totalPrice == Column!double([447.0, 799.0, 798.0, 299.0, 490.0]));
446 
447     Column!double discounts;
448     foreach(name; df.name)
449     {
450         if (name == "A")
451             discounts ~= 0.05;
452         else if (name == "C")
453             discounts ~= 0.02;
454         else
455             discounts ~= 0;
456     }
457 
458     df.totalPrice = (df.price - discounts) * df.quantity;
459     // TODO: Fix approxEqual to fix the below test
460     //assert(df.totalPrice == Column!double([446.85, 799, 797.96, 299, 490]));
461 
462     import std.algorithm;
463 
464     assert(df.rows
465            .filter!(r => r.name == "A" || r.name == "B")
466            .map!(r => r.quantity)
467            .sum == 4);
468 
469     df.add(Item("A", 149.0, 10));
470 
471     // Sort by name and then by quantity
472     auto result = df.rows
473         .multiSort!("a.name < b.name", "a.quantity > b.quantity", SwapStrategy.unstable);
474 
475     assert(result[0].name == "A" && result[1].name == "A");
476     assert(result[0].quantity == 10.0 && result[1].quantity == 3.0);
477     assert(result[2].name == "B");
478     assert(result[3].name == "C");
479     assert(result[4].name == "D");
480     assert(result[5].name == "E");
481 
482     // Access the Row column by label
483     assert(result[0]["name"].get!string == "A");
484 
485     // Access the Row column by index
486     assert(result[0][0].get!string == "A");
487 
488     struct PriceList
489     {
490         string name;
491         double price;
492     }
493 
494     auto items = df.rows
495         .sort!("a.name < b.name")
496         .uniq!("a.name == b.name")
497         .toDataFrame!PriceList;
498 
499     assert(items.length == 5);
500     assert(items.ncol == 2);
501     assert(items.columnNames == ["name", "price"]);
502 
503     auto dfCopy = df.toDataFrame;
504     assert(dfCopy.ncol == 4);
505     assert(dfCopy.nrow == 6);
506 
507     auto itemsDf = df.toDataFrame!PriceList;
508     assert(itemsDf.ncol == 2);
509     assert(itemsDf.nrow == 6);
510 
511     auto firstTwo = df.head(2);
512     assert(firstTwo.length == 2);
513     assert(firstTwo[0].name == "A");
514     assert(firstTwo[1].name == "B");
515 
516     auto lastTwo = df.tail(2);
517     assert(lastTwo.length == 2);
518     assert(lastTwo[0].name == "E");
519     assert(lastTwo[1].name == "A");
520 
521     // Access the DataFrame column by key
522     auto names1 = df["name"].get!(Column!string);
523     assert(names1[0] == "A");
524     auto names2 = df[0].get!(Column!string);
525     assert(names2[0] == "A");
526 
527     import std.stdio;
528 
529     struct Sample
530     {
531         string name;
532         string name2;
533         double value1;
534         double value2;
535         size_t count;
536     }
537 
538     auto df2 = new DataFrame!Sample(
539         name: ["A", "A", "B", "B", "B", "C"],
540         name2: ["A1", "A2", "B1", "B2", "B3", "C1"],
541         value1: [1, 2, 3, 4, 5, 6],
542         value2: [10, 20, 30, 40, 50, 60],
543         count: [0, 0, 0, 0, 0, 0]
544     );
545 
546     auto grouped = df2.rows.chunkBy!((a, b) => a.name == b.name);
547     auto logic = ["name": "first", "value1": "max", "value2": "sum", "name2": "last", "count": "count"];
548     auto df3 = df2.resample(grouped, logic);
549     assert(df3.nrow == 3);
550     assert(df3.name.values == ["A", "B", "C"]);
551     assert(df3.name2.values == ["A2", "B3", "C1"]);
552     assert(df3.value1.values == [2.0, 5.0, 6.0]);
553     assert(df3.value2.values == [30.0, 120.0, 60.0]);
554     assert(df3.count.values == [2, 3, 1]);
555 
556     auto df4 = df2.resample(df2.rows.chunkBy!((a, b) => a.name == b.name), "sum");
557     assert(df4.nrow == 3);
558     assert(df4.name.values == ["", "", ""]);
559     assert(df4.name2.values == ["", "", ""]);
560     assert(df4.value1.values == [3.0, 12.0, 6.0]);
561     assert(df4.value2.values == [30.0, 120.0, 60.0]);
562     assert(df4.count.values == [0, 0, 0]);
563 
564     auto df5 = df2.resample(df2.rows.chunkBy!((a, b) => a.name == b.name), ["first", "last", "max", "sum"]);
565     assert(df5.nrow == 3);
566     assert(df5.name.values == ["A", "B", "C"]);
567     assert(df5.name2.values == ["A2", "B3", "C1"]);
568     assert(df5.value1.values == [2.0, 5.0, 6.0]);
569     assert(df5.value2.values == [30.0, 120.0, 60.0]);
570     assert(df5.count.values == [0, 0, 0]);
571 
572     auto jsonOutput = `[{"count":0,"name":"A","name2":"A2","value1":2.0,"value2":30.0},
573                         {"count":0,"name":"B","name2":"B3","value1":5.0,"value2":120.0},
574                         {"count":0,"name":"C","name2":"C1","value1":6.0,"value2":60.0}]`;
575     assert(df5.toJSON == parseJSON(jsonOutput));
576 
577     struct Sample2
578     {
579         string name;
580         double value;
581         double avg;
582     }
583 
584     auto dfSample2 = new DataFrame!Sample2(
585         name: ["A", "A", "B", "B", "B", "C"],
586         value: [1, 2, 3, 4, 5, 6]
587     );
588 
589     import std.math : isNaN;
590 
591     dfSample2.avg = dfSample2.value.rolling!(mean, double)(2);
592     assert(dfSample2.avg[0].isNaN);
593     assert(dfSample2.avg.values[1..$] == [1.5, 2.5, 3.5, 4.5, 5.5]);
594 
595     double smaFunc(Row!Sample2[] data)
596     {
597         // Get the mean of the field "value"
598         return data.map!(a => a.value).mean;
599     }
600 
601     dfSample2.avg = dfSample2.rolling!(smaFunc, double)(2);
602     assert(dfSample2.avg[0].isNaN);
603     assert(dfSample2.avg.values[1..$] == [1.5, 2.5, 3.5, 4.5, 5.5]);
604 }