| 21 | public static final int STAGES = 5; |
| 22 | |
| 23 | public static void main(String[] args) throws Exception { |
| 24 | LeaveOneOutValidator.FORCE_SINGLE_THREADED = true; // need this when we compare results file by file |
| 25 | LangDescriptor[] languages = new LangDescriptor[]{ |
| 26 | QUORUM_DESCR, |
| 27 | // JAVA_DESCR, |
| 28 | // JAVA8_DESCR, |
| 29 | // ANTLR4_DESCR, |
| 30 | // SQLITE_CLEAN_DESCR, |
| 31 | // TSQL_CLEAN_DESCR, |
| 32 | }; |
| 33 | |
| 34 | Map<String, List<Float>> results = new HashMap<>(); |
| 35 | for (LangDescriptor language : languages) { |
| 36 | List<Float> errorRates = checkStability(language); |
| 37 | System.out.println(language.name+" "+errorRates); |
| 38 | results.put(language.name, errorRates); |
| 39 | } |
| 40 | for (String name : results.keySet()) { |
| 41 | System.out.println(name+" = "+results.get(name)); |
| 42 | } |
| 43 | |
| 44 | String python = |
| 45 | "#\n"+ |
| 46 | "# AUTO-GENERATED FILE. DO NOT EDIT\n" + |
| 47 | "# CodeBuff <version> '<date>'\n" + |
| 48 | "#\n"+ |
| 49 | "import numpy as np\n"+ |
| 50 | "import matplotlib.pyplot as plt\n\n" + |
| 51 | "import matplotlib\n" + |
| 52 | "fig = plt.figure()\n"+ |
| 53 | "ax = plt.subplot(111)\n"+ |
| 54 | "N = <N>\n" + |
| 55 | "sizes = range(0,N)\n" + |
| 56 | "<results:{r |\n" + |
| 57 | "<r> = [<results.(r); separator={,}>]\n"+ |
| 58 | "ax.plot(sizes, <r>, label=\"<r>\", marker='o')\n" + |
| 59 | "}>\n" + |
| 60 | "ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)\n" + |
| 61 | "xa = ax.get_xaxis()\n"+ |
| 62 | "xa.set_major_locator(matplotlib.ticker.MaxNLocator(integer=True))\n" + |
| 63 | "ax.set_xlabel(\"Formatting Stage; stage 0 is first formatting pass\")\n"+ |
| 64 | "ax.set_ylabel(\"Median Leave-one-out Validation Error Rate\")\n" + |
| 65 | "ax.set_title(\"<N>-Stage Formatting Stability\\nStage $n$ is formatted output of stage $n-1$\")\n"+ |
| 66 | "plt.legend()\n" + |
| 67 | "plt.tight_layout()\n" + |
| 68 | "fig.savefig('images/stability.pdf', format='pdf')\n"+ |
| 69 | "plt.show()\n"; |
| 70 | ST pythonST = new ST(python); |
| 71 | pythonST.add("results", results); |
| 72 | pythonST.add("version", version); |
| 73 | pythonST.add("date", new Date()); |
| 74 | pythonST.add("N", STAGES+1); |
| 75 | String fileName = "python/src/stability.py"; |
| 76 | Utils.writeFile(fileName, pythonST.render()); |
| 77 | System.out.println("wrote python code to "+fileName); |
| 78 | } |
| 79 | |
| 80 | public static List<Float> checkStability(LangDescriptor language) throws Exception { |