diff options
| author | Dibyendu Majumdar <mobile@majumdar.org.uk> | 2023-08-19 08:52:57 -0700 |
|---|---|---|
| committer | Dibyendu Majumdar <mobile@majumdar.org.uk> | 2025-06-13 15:42:08 -0700 |
| commit | 8e8dcf86abfdd8d90b11ede7db2e94666f4b7356 (patch) | |
| tree | cfcc75008c3dee594940af61f22dda12faf27d1c | |
| download | Simple-8e8dcf86abfdd8d90b11ede7db2e94666f4b7356.tar.gz Simple-8e8dcf86abfdd8d90b11ede7db2e94666f4b7356.zip | |
Chapter 1: Introduction
Co-authored-by: Cliff Click <cliffc@acm.org>
Co-authored-by: Thalia Archibald <thalia@archibald.dev>
Co-authored-by: hadha <helsgithub15@gmail.com>
Co-authored-by: XmiliaH <45106915+XmiliaH@users.noreply.github.com>
Co-authored-by: Tamás Hadházy <85063808+Hels15@users.noreply.github.com>
27 files changed, 1796 insertions, 0 deletions
diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 00000000..74fde018 --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,6 @@ +;;; Directory Local Variables +;;; For more information see (info "(emacs) Directory Variables") +( + (dired-mode . ((c-basic-offset . 4))) + (java-mode . ((c-basic-offset . 4))) +) diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..56d85b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Java stuff +# Compiled class file +*.class +build/ +*/docs/examples/*.o +*/docs/examples/*.exe + +# Log file +*.log + +# Idea +.idea/ +target/ +*.iml + +# Editors +#* +*~ +TAGS + +# Libs +*/lib/*jar diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..28574bca --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# Simple +A Simple showcase for the Sea-of-Nodes compiler IR + +This repo is intended to demonstrate the Sea-of-Nodes compiler IR. + +The Sea-of-Nodes is the core IR inside of HotSpot's C2 compiler +and Google's V8 compiler and Sun/Oracle's Graal compiler. + +Since we are show casing the SoN IR, the *language* being implemented is less +important. We're using a very simple language similar to C or Java, but with +far fewer features. Simple is strongly typed, object-oriented, with first- +class *functions* not *closures*. Object references are pointers, and null +pointer exceptions are disallowed by the typing system. Arrays will probably +be range-checked at some point, making Simple a fully safe language. Simple +has a minimal syntax that can be parsed with a recursive descent parser. + +The Sea-of-Nodes is used for machine code generation in these industrial +strength systems - but for this demonstration the backend is both difficult and +less important. This repo targets X86, RISC5 and ARM64 at least with +ahead-of-time compilation - but with an eye towards JIT compilation. + +This repo also is not intended to be a complete language in any sense, and so +the backend starts with levering Java: the Evaluator (first appears in Chapter +10) directly slowly interprets the SoN IR. Code-gen first appears in Chapter +19. + + +## Chapters + +The following is a rough plan, subject to change. + +Each chapter will be self-sufficient and complete; in the sense that each +chapter will fully implement a subset of the Simple language, and include +everything that was created in the previous chapter. Each chapter will also +include a detailed commentary on relevant aspects of the Sea Of Nodes +intermediate representation. + +The Simple language is styled after a subset of C or Java. + +* [Chapter 1](docs/chapter01/README.md): Script that returns an integer literal, i.e., an empty function that takes no arguments and returns a single integer value. The `return` statement. +* Chapter 2: Simple binary arithmetic such as addition, subtraction, multiplication, division + with constants. Peephole optimization / simple constant folding. +* Chapter 3: Local variables, and assignment statements. Read on RHS, SSA, more peephole optimization if local is a + constant. +* Chapter 4: A non-constant external variable input + named `arg`. Binary and Comparison operators involving constants and `arg`. + Non-zero values will be truthy. Peephole optimizations involving algebraic + simplifications. +* Chapter 5: `if` statement. CFG construction. +* Chapter 6: Peephole optimization around dead control flow. +* Chapter 7: `while` statement; looping constructs - eager phi approach. +* Chapter 8: Looping constructs continued, lazy phi creation, `break` and `continue` statements. +* Chapter 9: Global Value Numbering. Iterative peepholes to fixpoint. Worklists. +* Chapter 10: User defined Struct types. Memory effects: + general memory edges in SSA. Equivalence class aliasing. Null pointer + analysis. Peephole optimization around load-after-store/store-after-store. +* Chapter 11: Global Code Motion - Scheduling. +* Chapter 12: Float type. +* Chapter 13: Nested references in Structs. +* Chapter 14: Narrow primitive types (e.g. bytes) +* Chapter 15: One dimensional static length array type, with array loads and stores. +* Chapter 16: Constructors +* Chapter 17: Mutability & Syntax Sugar: `var`, `val`, `x+=y`, `for(init; test; next) body` +* Chapter 18: Functions and calls. +* Chapter 19: Instruction selection and portable compilation +* Chapter 20: Graph Coloring Register Allocation +* Chapter 21: Instruction Encodings & ELF +* Chapter 22: A Simple Hello, World! +* Chapter 23: Strings, Methods diff --git a/docs/chapter01/01-grammar.md b/docs/chapter01/01-grammar.md new file mode 100644 index 00000000..c5ddae2c --- /dev/null +++ b/docs/chapter01/01-grammar.md @@ -0,0 +1,26 @@ +# Grammar for Chapter 1 + +```antlrv4 +grammar SimpleLanguage; + +program + : statement EOF + ; + +statement + : 'return' expression ';' + ; + +expression + : primaryExpression + ; + +primaryExpression + : INTEGER_LITERAL + ; + +INTEGER_LITERAL + : [1-9][0-9]* + | [0] + ; +``` diff --git a/docs/chapter01/01-graph.gv b/docs/chapter01/01-graph.gv new file mode 100644 index 00000000..df256f52 --- /dev/null +++ b/docs/chapter01/01-graph.gv @@ -0,0 +1,17 @@ +digraph chapter01 { +/* +return 1; +*/ + rankdir=BT; + ordering="in"; + concentrate="true"; + subgraph cluster_Nodes { + Start1 [ shape=box style=filled fillcolor=yellow label="Start" ]; + Con_2 [ label="#1" ]; + Return3 [ shape=box style=filled fillcolor=yellow label="Return" ]; + } + edge [ fontname=Helvetica, fontsize=8 ]; + Con_2 -> Start1[taillabel=0 style=dotted]; + Return3 -> Start1[taillabel=0 color=red]; + Return3 -> Con_2[taillabel=1]; +} diff --git a/docs/chapter01/01-graph.svg b/docs/chapter01/01-graph.svg new file mode 100644 index 00000000..7f511966 --- /dev/null +++ b/docs/chapter01/01-graph.svg @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<!-- Generated by graphviz version 2.43.0 (0) + --> +<!-- Title: chapter01 Pages: 1 --> +<svg width="114pt" height="220pt" + viewBox="0.00 0.00 114.00 220.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> +<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 216)"> +<title>chapter01</title> +<polygon fill="white" stroke="transparent" points="-4,4 -4,-216 110,-216 110,4 -4,4"/> +<g id="clust1" class="cluster"> +<title>cluster_Nodes</title> +<polygon fill="none" stroke="black" points="8,-8 8,-204 98,-204 98,-8 8,-8"/> +</g> +<!-- Start1 --> +<g id="node1" class="node"> +<title>Start1</title> +<polygon fill="yellow" stroke="black" points="80,-196 26,-196 26,-160 80,-160 80,-196"/> +<text text-anchor="middle" x="53" y="-174.3" font-family="Times,serif" font-size="14.00">Start</text> +</g> +<!-- Con_2 --> +<g id="node2" class="node"> +<title>Con_2</title> +<ellipse fill="none" stroke="black" cx="43" cy="-106" rx="27" ry="18"/> +<text text-anchor="middle" x="43" y="-102.3" font-family="Times,serif" font-size="14.00">#1</text> +</g> +<!-- Con_2->Start1 --> +<g id="edge1" class="edge"> +<title>Con_2->Start1</title> +<path fill="none" stroke="black" stroke-dasharray="1,5" d="M45.42,-123.95C46.52,-131.65 47.85,-140.97 49.09,-149.64"/> +<polygon fill="black" stroke="black" points="45.65,-150.32 50.53,-159.72 52.58,-149.33 45.65,-150.32"/> +<text text-anchor="middle" x="42.42" y="-126.55" font-family="Helvetica,sans-Serif" font-size="8.00">0</text> +</g> +<!-- Return3 --> +<g id="node3" class="node"> +<title>Return3</title> +<polygon fill="yellow" stroke="black" points="86,-52 20,-52 20,-16 86,-16 86,-52"/> +<text text-anchor="middle" x="53" y="-30.3" font-family="Times,serif" font-size="14.00">Return</text> +</g> +<!-- Return3->Start1 --> +<g id="edge2" class="edge"> +<title>Return3->Start1</title> +<path fill="none" stroke="red" d="M64.05,-52.39C69.74,-62.38 76.08,-75.42 79,-88 82.62,-103.59 82.62,-108.41 79,-124 76.9,-133.04 73.03,-142.32 68.91,-150.53"/> +<polygon fill="red" stroke="red" points="65.69,-149.14 64.05,-159.61 71.86,-152.45 65.69,-149.14"/> +<text text-anchor="middle" x="61.05" y="-54.99" font-family="Helvetica,sans-Serif" font-size="8.00">0</text> +</g> +<!-- Return3->Con_2 --> +<g id="edge3" class="edge"> +<title>Return3->Con_2</title> +<path fill="none" stroke="black" d="M50.53,-52.3C49.43,-60.02 48.1,-69.29 46.87,-77.89"/> +<polygon fill="black" stroke="black" points="43.39,-77.5 45.44,-87.9 50.32,-78.49 43.39,-77.5"/> +<text text-anchor="middle" x="47.53" y="-54.9" font-family="Helvetica,sans-Serif" font-size="8.00">1</text> +</g> +</g> +</svg> diff --git a/docs/chapter01/01-use-def.gv b/docs/chapter01/01-use-def.gv new file mode 100644 index 00000000..d0e84ecb --- /dev/null +++ b/docs/chapter01/01-use-def.gv @@ -0,0 +1,7 @@ +digraph chapter01 { + rankdir=BT; + subgraph cluster_Nodes { + A, B; + } + A -> B; +} diff --git a/docs/chapter01/01-use-def.svg b/docs/chapter01/01-use-def.svg new file mode 100644 index 00000000..644ab5a2 --- /dev/null +++ b/docs/chapter01/01-use-def.svg @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<!-- Generated by graphviz version 2.43.0 (0) + --> +<!-- Title: chapter01 Pages: 1 --> +<svg width="94pt" height="148pt" + viewBox="0.00 0.00 94.00 148.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> +<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 144)"> +<title>chapter01</title> +<polygon fill="white" stroke="transparent" points="-4,4 -4,-144 90,-144 90,4 -4,4"/> +<g id="clust1" class="cluster"> +<title>cluster_Nodes</title> +<polygon fill="none" stroke="black" points="8,-8 8,-132 78,-132 78,-8 8,-8"/> +</g> +<!-- A --> +<g id="node1" class="node"> +<title>A</title> +<ellipse fill="none" stroke="black" cx="43" cy="-34" rx="27" ry="18"/> +<text text-anchor="middle" x="43" y="-30.3" font-family="Times,serif" font-size="14.00">A</text> +</g> +<!-- B --> +<g id="node2" class="node"> +<title>B</title> +<ellipse fill="none" stroke="black" cx="43" cy="-106" rx="27" ry="18"/> +<text text-anchor="middle" x="43" y="-102.3" font-family="Times,serif" font-size="14.00">B</text> +</g> +<!-- A->B --> +<g id="edge1" class="edge"> +<title>A->B</title> +<path fill="none" stroke="black" d="M43,-52.3C43,-60.02 43,-69.29 43,-77.89"/> +<polygon fill="black" stroke="black" points="39.5,-77.9 43,-87.9 46.5,-77.9 39.5,-77.9"/> +</g> +</g> +</svg> diff --git a/docs/chapter01/Makefile b/docs/chapter01/Makefile new file mode 100644 index 00000000..12659b8f --- /dev/null +++ b/docs/chapter01/Makefile @@ -0,0 +1,12 @@ +GV := $(wildcard *.gv) +SVG := $(patsubst %.gv,%.svg,$(GV)) + +.PHONY: all +all: $(SVG) + +%.svg: %.gv + dot -Tsvg $< > $@ + +.PHONY: clean +clean: + @rm -f $(SVG) diff --git a/docs/chapter01/README.md b/docs/chapter01/README.md new file mode 100644 index 00000000..5afe48b8 --- /dev/null +++ b/docs/chapter01/README.md @@ -0,0 +1,187 @@ +# Chapter 1: Introduction + +# Table of Contents + +1. [Implementation Language](#implementation-language) +2. [Assumptions](#assumptions) +3. [Architecture](#architecture) +4. [Data Structures](#data-structures) +5. [Intermediate Representation as a Graph of Nodes](#intermediate-representation-as-a-graph-of-nodes) +6. [Nodes are in a Graph](#nodes-are-in-a-graph) +7. [Types of Nodes](#types-of-nodes) +8. [Unique Node ID](#unique-node-id) +9. [Start Node](#start-node) +10. [Constant Node](#constant-node) +11. [Return Node](#return-node) +12. [Representation](#representation) + + +You can also read [this chapter](https://github.com/SeaOfNodes/Simple/tree/linear-chapter01) in a linear Git revision history on the [linear](https://github.com/SeaOfNodes/Simple/tree/linear) branch. + + +In this chapter we aim to compile simple scripts such as: + +``` +return 1; +``` + +We implement the `return` statement. +The `return` statement accepts an `expression` as an argument. +The only `expression` type is an integer literal, as shown in the example above. + +To implement this simple language, we introduce a few key components and data structures. + +Here is the [complete language grammar](01-grammar.md) for this chapter. + + +## Implementation Language + +Our implementation language is Java. We chose Java as it is widely available and understood. + +## Assumptions + +We assume that the reader is familiar with traditional linear intermediate representations, and is familiar with terms such as Basic Block, Control Flow Graph, etc. No attempt is made to explain these topics. +If necessary the reader can consult a standard compiler text book. + +## Architecture + +We construct the intermediate Sea of Nodes (SoN) representation directly as we parse the language. There is no Abstract Syntax Tree representation. The reason for this is to demonstrate a key benefit of the SoN IR: +a number of pessimistic peephole optimizations can be performed while parsing a language. This aspect is more fully explored from [Chapter 2](../chapter02/README.md) onwards. + +## Data Structures + +Our data structures are based upon the descriptions provided in following papers: + +* [From Quads to Graphs: An Intermediate Representation's Journey](http://softlib.rice.edu/pub/CRPC-TRs/reports/CRPC-TR93366-S.pdf) +* [Combining Analyses, Combining Optimizations](https://dl.acm.org/doi/pdf/10.1145/201059.201061) +* [A Simple Graph-Based Intermediate Representation](https://www.oracle.com/technetwork/java/javase/tech/c2-ir95-150110.pdf) +* [Global Code Motion Global Value Numbering](https://courses.cs.washington.edu/courses/cse501/06wi/reading/click-pldi95.pdf) +* [EasySSA](https://www.dropbox.com/scl/fi/0ww4sgl3ynep9hhe3i4xn/EasySSA.pdf?rlkey=2cp78hzxke62flkmyneiebzoz&dl=0) +* [SeaOfNodes](https://www.dropbox.com/scl/fi/cxykfvlzsmlcatyg6rlbt/SeaOfNodes.pdf?rlkey=z6o7y3rwr6atrejilcze6r8x0&e=1&dl=0) + +Following the lead from above, we represent our intermediate representation using an object oriented data model. Details of the +representation follow. + +### Intermediate Representation as a Graph of Nodes + +The intermediate representation is a graph of Node objects. The `Node` class is the base type for objects in the IR graph. +The `Node` class provides common capabilities that are inherited by all subtypes. +Each subtype implements semantics relevant to that subtype. + +Each `Node` represents an "instruction" as it may appear in traditional IRs. + +### Nodes are in a Graph + +The key idea of the Sea of Nodes IR is that each Node is linked to other Nodes by def-use dependencies. +As this is such an important and fundamental aspect of the IR, it is important to understand how we implement this, and depict in graph visuals. + +The base `Node` class maintains a list of Nodes that are inputs to it. An input is an edge from a "def" to a "use". What this means is that if `B` is definition, and `A` uses `B`, +then there is a def-use edge from `B` to `A`. + +Visually we show an arrow from the "use" to the "def". Here is an example: + + + +From an implementation point of view, our `Node` type also maintains a reverse link. +This means that in the above scenario: + +* Since `A` is a "use" of `B`, then `B` will appear in `A`'s list of inputs. +* Conversely, `B` maintains a list of outputs, and `A` will appear in this list. +* A key invariant is that these edges must be in sync at all times. + +Here is how this looks in the `Node` class: + +```java +public abstract class Node { + + /** + * Inputs to the node. These are use-def references to Nodes. + * <p> + * Generally fixed length, ordered, nulls allowed, no unused trailing space. + * Ordering is required because e.g. "a/b" is different from "b/a". + * The first input (offset 0) is often a Control node. + */ + public final ArrayList<Node> _inputs; + + /** + * Outputs reference Nodes that are not null and have this Node as an + * input. These nodes are users of this node, thus these are def-use + * references to Nodes. + * <p> + * Outputs directly match inputs, making a directed graph that can be + * walked in either direction. These outputs are typically used for + * efficient optimizations but otherwise have no semantics meaning. + */ + public final ArrayList<Node> _outputs; +} +``` + +### Types of Nodes + +There are two categories of Nodes in the intermediate representation. + +* **Control Nodes** - these represent the control flow subgraph (CFG) of the compiled program +* **Data Nodes** - these capture the data semantics + +The following control and data nodes appear in this chapter. + +| Node Name | Type | Description | Inputs | Value | +|-----------|---------|-----------------------------------------------|------------------------------------------------------------------|-------------------------------------------------------| +| Start | Control | Start of function | None | None for now as we do not have function arguments yet | +| Return | Control | Represents the termination of a function | Predecessor control node, Data node value | Return value of the function | +| Constant | Data | Represents constants such as integer literals | None, however Start node is set as input to enable graph walking | Value of the constant | + +Within a traditional basic block, instructions are executed in sequence. In the Sea of Nodes model, the correct sequence of instructions is determined by a scheduling +algorithm that depends only on dependencies between nodes (including control dependencies) that are explicit as edges in the graph. This enables a number of optimizations +at very little cost (nearly always small constant time) because all dependencies are always available. + +### Unique Node ID + +Each node is assigned a unique dense integer Node ID when created. This ID is +useful for debugging, efficiently computing equality and e.g. as an index into +a bit vector, which in turn is used to efficiently visit a (possibly cyclic) +graph. We discuss Node equality in [Chapter 9](../chapter09/README.md). + +### Start Node + +The Start node represents the start of the function. For now, we do not have any inputs to Start because our function does not +yet accept parameters. When we add parameters the value of Start will be a tuple, and will require Projections to extract the values. +We discuss this in detail in [Chapter 4](../chapter04/README.md). + +### Constant Node + +A Constant node represents a constant value. At present, the only constants +that we allow are integer literals; therefore Constants contain an integer +value. As we add other types of constants, we will refactor how we represent +Constants. + +Constants have no semantic inputs. However, we set Start as an input to +Constants to enable a forward graph walk. This edge carries no semantic +meaning, and it is present _solely_ to allow visitation. + +The Constant's value is the value stored in it. + +### Return Node + +The Return node has two inputs. The first input is a control node and the +second is the data node that supplies the return value. + +In this presentation, Return functions as a Stop node, since multiple `return` statements are not possible. +The Stop node will be introduced in [Chapter 5](../chapter05/README.md) when we implement `if` statements. + +The Return's output is the value from the data node. + +### Representation + +Here is visualization of the program: + +``` +return 1; +``` + + + +* Control nodes appear as square boxes with yellow background +* Control edges are in bold red +* The edges from Constants to Start are shown in dotted lines as these are not true control edges +* We label each edge with its position in the `_inputs` array, thus `0` means the edge is `_inputs[0]`. diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..789ccf32 --- /dev/null +++ b/pom.xml @@ -0,0 +1,52 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.seaofnodes</groupId> + <artifactId>simple</artifactId> + <packaging>jar</packaging> + <version>1.0</version> + <name>SeaOfNodes Simple Programming Language</name> + <description>A simple language implementation to demonstrate Sea of Nodes IR</description> + <url>https://github.com/SeaOfNodes/Simple</url> + <licenses> + <license> + <name>Apache License Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0</url> + </license> + </licenses> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <maven.compiler.source>21</maven.compiler.source> + <maven.compiler.target>21</maven.compiler.target> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.13.2</version> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>3.1.2</version> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.10.1</version> + <configuration> + <release>21</release> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/src/main/java/com/seaofnodes/simple/Parser.java b/src/main/java/com/seaofnodes/simple/Parser.java new file mode 100644 index 00000000..806c985d --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/Parser.java @@ -0,0 +1,250 @@ +package com.seaofnodes.simple; + +import com.seaofnodes.simple.node.*; + +/** + * The Parser converts a Simple source program to the Sea of Nodes intermediate + * representation directly in one pass. There is no intermediate Abstract + * Syntax Tree structure. + * <p> + * This is a simple recursive descent parser. All lexical analysis is done here as well. + */ +public class Parser { + + /** + * A Global Static, unique to each compilation. This is a public, so we + * can make constants everywhere without having to thread the StartNode + * through the entire parser and optimizer. + * <p> + * To make the compiler multithreaded, this field will have to move into a TLS. + */ + public static StartNode START; + + // The Lexer. Thin wrapper over a byte[] buffer with a cursor. + private final Lexer _lexer; + + + public Parser(String source) { + _lexer = new Lexer(source); + Node.reset(); + START = new StartNode(); + } + + public ReturnNode parse() { + var ret = (ReturnNode) parseStatement(); + if (!_lexer.isEOF()) throw error("Syntax error, unexpected " + _lexer.getAnyNextToken()); + return ret; + } + + /** + * Parses a statement + * + * <pre> + * returnStatement + * </pre> + * @return a {@link Node} or {@code null} + */ + private Node parseStatement() { + if (matchx("return")) return parseReturn(); + throw errorSyntax("a statement"); + } + + /** + * Parses a return statement; "return" already parsed. + * + * <pre> + * 'return' expr ; + * </pre> + * @return an expression {@link Node}, never {@code null} + */ + private ReturnNode parseReturn() { + var expr = require(parseExpression(), ";"); + return new ReturnNode(START, expr); + } + + /** + * Parse an expression of the form: + * + * <pre> + * expr : primaryExpr + * </pre> + * @return an expression {@link Node}, never {@code null} + */ + private Node parseExpression() { + return parsePrimary(); + } + + /** + * Parse a primary expression: + * + * <pre> + * primaryExpr : integerLiteral + * </pre> + */ + private Node parsePrimary() { + _lexer.skipWhiteSpace(); + if (_lexer.isNumber()) + return parseIntegerLiteral(); + throw error("Syntax error, expected integer literal"); + } + + /** + * Parse integer literal + * + * <pre> + * integerLiteral: [1-9][0-9]* | [0] + * </pre> + */ + private ConstantNode parseIntegerLiteral() { + return new ConstantNode(_lexer.parseNumber()); + } + + ////////////////////////////////// + // Utilities for lexical analysis + + // Return true and skip if "syntax" is next in the stream. + private boolean match (String syntax) { return _lexer.match (syntax); } + // Match must be "exact", not be followed by more id letters + private boolean matchx(String syntax) { return _lexer.matchx(syntax); } + + // Require an exact match + private void require(String syntax) { require(null, syntax); } + private Node require(Node n, String syntax) { + if (match(syntax)) return n; + throw errorSyntax(syntax); + } + + RuntimeException errorSyntax(String syntax) { + return error("Syntax error, expected " + syntax + ": " + _lexer.getAnyNextToken()); + } + + static RuntimeException error(String errorMessage) { + return new RuntimeException(errorMessage); + } + + //////////////////////////////////// + // Lexer components + + private static class Lexer { + + // Input buffer; an array of text bytes read from a file or a string + private final byte[] _input; + // Tracks current position in input buffer + private int _position = 0; + + /** + * Record the source text for lexing + */ + public Lexer(String source) { + this(source.getBytes()); + } + + /** + * Direct from disk file source + */ + public Lexer(byte[] buf) { + _input = buf; + } + + // True if at EOF + private boolean isEOF() { + return _position >= _input.length; + } + + // Peek next character, or report EOF + private char peek() { + return isEOF() ? Character.MAX_VALUE // Special value that causes parsing to terminate + : (char) _input[_position]; + } + + private char nextChar() { + char ch = peek(); + _position++; + return ch; + } + + // True if a white space + private boolean isWhiteSpace() { + return peek() <= ' '; // Includes all the use space, tab, newline, CR + } + + /** + * Return the next non-white-space character + */ + private void skipWhiteSpace() { + while (isWhiteSpace()) _position++; + } + + + // Return true, if we find "syntax" after skipping white space; also + // then advance the cursor past syntax. + // Return false otherwise, and do not advance the cursor. + boolean match(String syntax) { + skipWhiteSpace(); + int len = syntax.length(); + if (_position + len > _input.length) return false; + for (int i = 0; i < len; i++) + if ((char) _input[_position + i] != syntax.charAt(i)) + return false; + _position += len; + return true; + } + + boolean matchx(String syntax) { + if( !match(syntax) ) return false; + if( !isIdLetter(peek()) ) return true; + _position -= syntax.length(); + return false; + } + + // Used for errors + String getAnyNextToken() { + if (isEOF()) return ""; + if (isIdStart(peek())) return parseId(); + if (isNumber(peek())) return parseNumberString(); + if (isPunctuation(peek())) return parsePunctuation(); + return String.valueOf(peek()); + } + + boolean isNumber() {return isNumber(peek());} + boolean isNumber(char ch) {return Character.isDigit(ch);} + + private long parseNumber() { + String snum = parseNumberString(); + if (snum.length() > 1 && snum.charAt(0) == '0') + throw error("Syntax error: integer values cannot start with '0'"); + return Long.parseLong(snum); + } + private String parseNumberString() { + int start = _position; + while (isNumber(nextChar())) ; + return new String(_input, start, --_position - start); + } + + // First letter of an identifier + private boolean isIdStart(char ch) { + return Character.isAlphabetic(ch) || ch == '_'; + } + + // All characters of an identifier, e.g. "_x123" + private boolean isIdLetter(char ch) { + return Character.isLetterOrDigit(ch) || ch == '_'; + } + + private String parseId() { + int start = _position; + while (isIdLetter(nextChar())) ; + return new String(_input, start, --_position - start); + } + + private boolean isPunctuation(char ch) { + return "=;[]<>(){}+-/*!".indexOf(ch) != -1; + } + + private String parsePunctuation() { + int start = _position; + return new String(_input, start, 1); + } + } + +} diff --git a/src/main/java/com/seaofnodes/simple/node/ConstantNode.java b/src/main/java/com/seaofnodes/simple/node/ConstantNode.java new file mode 100644 index 00000000..21f65084 --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/node/ConstantNode.java @@ -0,0 +1,25 @@ +package com.seaofnodes.simple.node; + +import com.seaofnodes.simple.Parser; + +/** + * A Constant node represents a constant value. At present, the only constants + * that we allow are integer literals; therefore Constants contain an integer + * value. As we add other types of constants, we will refactor how we represent + * Constants. + * <p> + * Constants have no semantic inputs. However, we set Start as an input to + * Constants to enable a forward graph walk. This edge carries no semantic + * meaning, and it is present <em>solely</em> to allow visitation. + * <p> + * The Constant's value is the value stored in it. + */ +public class ConstantNode extends Node { + + public final long _value; + + public ConstantNode(long value) { + super(Parser.START); + _value = value; + } +} diff --git a/src/main/java/com/seaofnodes/simple/node/Node.java b/src/main/java/com/seaofnodes/simple/node/Node.java new file mode 100644 index 00000000..5d74e206 --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/node/Node.java @@ -0,0 +1,77 @@ +package com.seaofnodes.simple.node; + +import java.util.*; + +/** + * All Nodes in the Sea of Nodes IR inherit from the Node class. + * The Node class provides common functionality used by all subtypes. + * Subtypes of Node specialize by overriding methods. + */ +public abstract class Node { + + /** + * Each node has a unique dense Node ID within a compilation context + * The ID is useful for debugging, for using as an offset in a bitvector, + * as well as for computing equality of nodes (to be implemented later). + */ + public final int _nid; + + /** + * Inputs to the node. These are use-def references to Nodes. + * <p> + * Generally fixed length, ordered, nulls allowed, no unused trailing space. + * Ordering is required because e.g. "a/b" is different from "b/a". + * The first input (offset 0) is often a {@link #isCFG} node. + */ + public final ArrayList<Node> _inputs; + + /** + * Outputs reference Nodes that are not null and have this Node as an + * input. These nodes are users of this node, thus these are def-use + * references to Nodes. + * <p> + * Outputs directly match inputs, making a directed graph that can be + * walked in either direction. These outputs are typically used for + * efficient optimizations but otherwise have no semantics meaning. + */ + public final ArrayList<Node> _outputs; + + /** + * A private Global Static mutable counter, for unique node id generation. + * To make the compiler multi-threaded, this field will have to move into a TLS. + * Starting with value 1, to avoid bugs confusing node ID 0 with uninitialized values. + * */ + private static int UNIQUE_ID = 1; + + protected Node(Node... inputs) { + _nid = UNIQUE_ID++; // allocate unique dense ID + _inputs = new ArrayList<>(); + Collections.addAll(_inputs,inputs); + _outputs = new ArrayList<>(); + for( Node n : _inputs ) + if( n != null ) + n._outputs.add( this ); + } + + /** + * Gets the ith input node + * @param i Offset of the input node + * @return Input node or null + */ + public Node in(int i) { return _inputs.get(i); } + + public int nIns() { return _inputs.size(); } + + public int nOuts() { return _outputs.size(); } + + public boolean isUnused() { return nOuts() == 0; } + + public boolean isCFG() { return false; } + + + /** + * Used to allow repeating tests in the same JVM. This just resets the + * Node unique id generator, and is done as part of making a new Parser. + */ + public static void reset() { UNIQUE_ID = 1; } +} diff --git a/src/main/java/com/seaofnodes/simple/node/ReturnNode.java b/src/main/java/com/seaofnodes/simple/node/ReturnNode.java new file mode 100644 index 00000000..85bbc83f --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/node/ReturnNode.java @@ -0,0 +1,23 @@ +package com.seaofnodes.simple.node; + +/** + * The Return node has two inputs. The first input is a control node and the + * second is the data node that supplies the return value. + * <p> + * In this presentation, Return functions as a Stop node, since multiple <code>return</code> statements are not possible. + * The Stop node will be introduced in Chapter 6 when we implement <code>if</code> statements. + * <p> + * The Return's output is the value from the data node. + */ +public class ReturnNode extends Node { + + public ReturnNode(Node ctrl, Node data) { + super(ctrl, data); + } + + public Node ctrl() { return in(0); } + public Node expr() { return in(1); } + + @Override public boolean isCFG() { return true; } + +} diff --git a/src/main/java/com/seaofnodes/simple/node/StartNode.java b/src/main/java/com/seaofnodes/simple/node/StartNode.java new file mode 100644 index 00000000..04bd6bb4 --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/node/StartNode.java @@ -0,0 +1,16 @@ +package com.seaofnodes.simple.node; + +/** + * The Start node represents the start of the function. For now, we do not have any inputs to Start because our function does not + * yet accept parameters. When we add parameters the value of Start will be a tuple, and will require Projections to extract the values. + * We discuss this in detail in Chapter 9: Functions and Calls. + */ +public class StartNode extends Node { + + public StartNode(/*arguments go here*/) { + super(); + } + + @Override public boolean isCFG() { return true; } + +} diff --git a/src/test/java/com/seaofnodes/simple/Chapter01Test.java b/src/test/java/com/seaofnodes/simple/Chapter01Test.java new file mode 100644 index 00000000..a91aabb1 --- /dev/null +++ b/src/test/java/com/seaofnodes/simple/Chapter01Test.java @@ -0,0 +1,107 @@ +package com.seaofnodes.simple; + +import com.seaofnodes.simple.node.*; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class Chapter01Test { + + @Test + public void testSimpleProgram() { + Parser parser = new Parser("return 1;"); + ReturnNode ret = parser.parse(); + StartNode start = Parser.START; + + assertEquals(start, ret.ctrl()); + Node expr = ret.expr(); + if( expr instanceof ConstantNode con ) { + assertEquals(start,con.in(0)); + assertEquals(1, con._value); + } else { + fail(); + } + } + + @Test + public void testZero() { + Parser parser = new Parser("return 0;"); + parser.parse(); + StartNode start = Parser.START; + for( Node use : start._outputs ) + if( use instanceof ConstantNode con ) + assertEquals(0,con._value); + } + + @Test + public void testBad1() { + try { + new Parser("ret").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error, expected a statement: ret",e.getMessage()); + } + } + + @Test + public void testBad2() { + try { + new Parser("return 0123;").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error: integer values cannot start with '0'",e.getMessage()); + } + } + + @Test + public void testBad3() { + try { + new Parser("return --12;").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error, expected integer literal",e.getMessage()); + } + } + + @Test + public void testBad4() { + try { + new Parser("return 100").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error, expected ;: ",e.getMessage()); + } + } + + // Negative numbers require unary operator support that is not in scope + @Test + public void testBad5() { + try { + new Parser("return -100;").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error, expected integer literal",e.getMessage()); + } + } + + @Test + public void testBad6() { + try { + new Parser("return100").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error, expected a statement: return100",e.getMessage()); + } + } + + @Test + public void testBad7() { + try { + new Parser("return 1;}").parse(); + fail(); + } catch( RuntimeException e ) { + assertEquals("Syntax error, unexpected }",e.getMessage()); + } + } + +} diff --git a/transpile-tests/Makefile b/transpile-tests/Makefile new file mode 100644 index 00000000..b2a6e74b --- /dev/null +++ b/transpile-tests/Makefile @@ -0,0 +1,93 @@ +SHELL := /bin/bash +.DELETE_ON_ERROR: + +# for printing variable values +# usage: make print-VARIABLE +# > VARIABLE = value_of_variable +print-% : ; @echo $* = $($*) + +# literal space +space := $() $() + +# Decide OS-specific questions +# jar-file seperator +ifeq ($(OS),Windows_NT) + SEP = ; +else + SEP = : +endif +# Find a reasonable ctags. +CTAGS = $(shell which ctags) +# Hack for MacOS: /usr/bin/ctags is unfriendly, so look for ctags from brew +ifeq ($(UNAME),Darwin) + CTAGS = $(shell brew list ctags 2> /dev/null | grep bin/ctags) +endif + +# Fun Args to javac. Mostly limit to java8 source definitions, and fairly +# aggressive lint warnings. +JAVAC_ARGS = -g + +# Source code +SIMPLE := com/seaofnodes/simple/transpile +SRC := src/main/java +TST := src/test/java +CLZDIR:= build/classes +main_javas := $(wildcard $(SRC)/$(SIMPLE)/*java $(SRC)/$(SIMPLE)/*/*java) +test_javas := $(wildcard $(TST)/$(SIMPLE)/*java $(TST)/$(SIMPLE)/*/*java) +main_classes := $(patsubst $(SRC)/%java,$(CLZDIR)/main/%class,$(main_javas)) +test_classes := $(patsubst $(TST)/%java,$(CLZDIR)/test/%class,$(test_javas)) +test_cp := $(patsubst $(TST)/$(SIMPLE)/%.java,com.seaofnodes.simple.transpile.%,$(wildcard $(TST)/$(SIMPLE)/*Test.java)) +classes = $(main_classes) $(test_classes) +# All the libraries +libs = $(wildcard lib/*jar) +jars = $(subst $(space),$(SEP),$(libs)) + + +default_targets := $(main_classes) $(test_classes) +# Optionally add ctags to the default target if a reasonable one was found. +ifneq ($(CTAGS),) +default_targets += tags +endif + +default: $(default_targets) + +# Compile just the out-of-date files +$(main_classes): build/classes/main/%class: $(SRC)/%java + @echo "compiling " $@ " because " $? + @[ -d $(CLZDIR)/main ] || mkdir -p $(CLZDIR)/main + @javac $(JAVAC_ARGS) -cp "$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(SRC) -d $(CLZDIR)/main $(main_javas) + +$(test_classes): $(CLZDIR)/test/%class: $(TST)/%java $(main_classes) + @echo "compiling " $@ " because " $? + @[ -d $(CLZDIR)/test ] || mkdir -p $(CLZDIR)/test + @javac $(JAVAC_ARGS) -cp "$(CLZDIR)/test$(SEP)$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(TST) -d $(CLZDIR)/test $(test_javas) + +# Base launch line for JVM tests +JVM=nice java -ea -cp "build/classes/main${SEP}${jars}${SEP}$(CLZDIR)/test" + +tests: $(default_targets) + @echo "testing " $(test_cp) + @$(JVM) org.junit.runner.JUnitCore $(test_cp) + +.PHONY: clean +clean: + rm -rf build + rm -f TAGS + (find . -name "*~" -exec rm {} \; 2>/dev/null; exit 0) + +# Download libs from maven +lib: lib/junit-4.12.jar lib/hamcrest-core-1.3.jar + +# Jars +lib/junit-4.12.jar: + @[ -d lib ] || mkdir -p lib + @(cd lib; wget https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar) + +lib/hamcrest-core-1.3.jar: + @[ -d lib ] || mkdir -p lib + @(cd lib; wget https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar) + +# Build emacs tags (part of a tasty emacs ide experience) +tags: $(main_javas) $(test_javas) + @rm -f TAGS + @$(CTAGS) -e --recurse=yes --extra=+q --fields=+fksaiS $(SRC) $(TST) diff --git a/transpile-tests/README.md b/transpile-tests/README.md new file mode 100644 index 00000000..aafd3161 --- /dev/null +++ b/transpile-tests/README.md @@ -0,0 +1,15 @@ +# Transpile Tests + +This module may be helpful if you want to create an implementation of Simple in another language. +It can parse the JUnit tests to automatically translate most of the boilerplate. +Some manual tweaks will still be required though, because only common patterns are detected +(e.g. 01: testSimpleProgram, testZero; 04: testVarArg; 08: testRegress5; 09: testMeet, ...) + +## Usage + +Just call `JUnitParser.parseRepository` to get a list of parsed test classes for each chapter, +and implement a pretty printer for your favourite language. + +For an example you can refer to `printers.RustPrinter`, +which produces tests similar to the ones used by +[Simple-Rust](https://github.com/SeaOfNodes/Simple-Rust). diff --git a/transpile-tests/pom.xml b/transpile-tests/pom.xml new file mode 100644 index 00000000..092e26ee --- /dev/null +++ b/transpile-tests/pom.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.seaofnodes</groupId> + <artifactId>simple</artifactId> + <version>1.0</version> + </parent> + <artifactId>transpile-tests</artifactId> + <packaging>jar</packaging> + <name>Transpile Tests</name> +</project> diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java new file mode 100644 index 00000000..19709b9e --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java @@ -0,0 +1,173 @@ +package com.seaofnodes.simple.transpile; + +import com.sun.source.tree.*; +import com.sun.source.util.TreeScanner; + +import java.util.ArrayList; +import java.util.HashMap; + +class AstVisitor extends TreeScanner<Void, Void> { + final HashMap<String, TestClass> results = new HashMap<>(); + + private String className; + private TestMethod current; + private boolean inCatch; + + @Override + public Void visitClass(ClassTree node, Void unused) { + var name = node.getSimpleName().toString(); + if (name.matches("Chapter\\d+Test")) { + var backup = className; + className = name; + super.visitClass(node, unused); + className = backup; + } + return null; + } + + @Override + public Void visitMethod(MethodTree node, Void unused) { + if (className != null) { + current = new TestMethod(); + current.name = node.getName().toString(); + super.visitMethod(node, unused); + results.compute(className, (k, v) -> { + if (v == null) + v = new TestClass(className, new ArrayList<>()); + v.methods().add(current); + return v; + }); + current = null; + } + return null; + } + + @Override + public Void visitMethodInvocation(MethodInvocationTree node, Void unused) { + if (current != null) { + var methodSelect = node.getMethodSelect().toString(); + var args = node.getArguments(); + + if (methodSelect.endsWith(".parse")) { + current.showAfterParse = switch (args.size()) { + case 0 -> false; + case 1 -> (Boolean) literal(args.getFirst()); + default -> throw new RuntimeException("unexpected parse arguments" + node); + }; + } + if (methodSelect.endsWith(".iterate")) { + current.iterate = true; + current.showAfterIterate = switch (args.size()) { + case 0 -> false; + case 1 -> (Boolean) literal(args.getFirst()); + default -> throw new RuntimeException("unexpected parse arguments" + node); + }; + } + if (methodSelect.endsWith("assertEquals")) { + if (inCatch) { + if (args.size() != 2) + throw new RuntimeException("Unexpected number of arguments " + node); + if (args.get(1).toString().contains("e.getMessage()")) { + current.parseErrorMessage = (String) literal(args.get(0)); + } + } else { + if (args.size() != 2) + throw new RuntimeException("Unexpected number of arguments " + node); + if (args.get(1).toString().contains("evaluate(")) { + // visit evaluate(..) or evaluate(..).toString() + var eval = args.get(1).accept(new TreeScanner<MethodInvocationTree, Void>() { + @Override + public MethodInvocationTree reduce(MethodInvocationTree r1, MethodInvocationTree r2) { + return r1 != null ? r1 : r2; + } + + @Override + public MethodInvocationTree visitMethodInvocation(MethodInvocationTree n, Void unused1) { + if (n.getMethodSelect().toString().endsWith(".evaluate")) + return n; + return super.visitMethodInvocation(n, unused1); + } + }, null); + + Object result = literal(args.get(0)); + var evalArgs = eval.getArguments(); + if (evalArgs.size() != 2 || !evalArgs.get(0).toString().equals("stop")) + throw new IllegalArgumentException("Unexpected eval arguments " + node); + long parameter = Long.parseLong(literal(evalArgs.get(1)).toString()); + current.evaluations.add(new TestMethod.Evaluation(result, parameter)); + } else if (args.get(1).toString().endsWith(".toString()") || args.get(1).toString().endsWith(".print()")) { + if (current.assertStopEquals == null) { + // Assume the first one prints the stop node. Ignore assertions from evaluator: + // assertEquals("int[] {\n # :int;\n [] :int;\n}",obj.struct().toString()); + current.assertStopEquals = (String) literal(args.get(0)); + } + } + } + } + if (methodSelect.endsWith("assertTrue")) { + var arg = args.getFirst().toString(); + current.assertStopRetCtrlIsProj = arg.contains("stop.ret().ctrl() instanceof ProjNode"); + current.assertStopRetCtrlIsCProj = arg.contains("stop.ret().ctrl() instanceof CProjNode"); + current.assertStopRetCtrlIsRegion = arg.contains("stop.ret().ctrl() instanceof RegionNode"); + } + if (methodSelect.endsWith("prettyPrint")) { + if (args.size() < 2 || args.size() > 3 || !args.get(0).toString().equals("stop") || !args.get(1).toString().equals("99")) + throw new RuntimeException("unexpected prettyPrint args " + node); + current.irPrinter = true; + current.irPrinterLLVM = args.size() == 3 && (Boolean) literal(args.get(2)); + } + } + return super.visitMethodInvocation(node, unused); // otherwise we would miss the first part of new Parser("").parse() + } + + @Override + public Void visitNewClass(NewClassTree node, Void unused) { + if (current != null) { + if (node.getIdentifier().toString().equals("Parser")) { + var args = node.getArguments(); + switch (args.size()) { + case 1 -> { + } + case 2 -> current.parserArg = switch (args.get(1).toString()) { + case "TypeInteger.BOT" -> new TestMethod.Arg.IntBot(); + case "TypeInteger.TOP" -> new TestMethod.Arg.IntTop(); + case String s -> { + var parts = s.split("[()]"); + if (parts.length != 2 | !parts[0].equals("TypeInteger.constant")) + throw new RuntimeException("Unexpected Argument " + s); + yield new TestMethod.Arg.IntConstant(Long.parseLong(parts[1])); + } + }; + default -> throw new RuntimeException("Parser constructor with unexpected arguments: " + node); + } + current.parserInput = (String) literal(args.get(0)); + } else if (node.getIdentifier().toString().equals("GraphVisualizer")) { + // only happens in Chapter02Tests + current.showAfterParse = true; + } + } + return super.visitNewClass(node, unused); + } + + @Override + public Void visitCatch(CatchTree node, Void unused) { + inCatch = true; + super.visitCatch(node, unused); + inCatch = false; + return null; + } + + @Override + public Void visitAssignment(AssignmentTree node, Void unused) { + if (current != null && node.getVariable().toString().equals("Node._disablePeephole")) { + if ((Boolean) literal(node.getExpression())) { + current.disablePeephole = true; + } + } + return super.visitAssignment(node, unused); + } + + private Object literal(ExpressionTree t) { + return ((LiteralTree) t).getValue(); + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java new file mode 100644 index 00000000..2db2a08c --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java @@ -0,0 +1,71 @@ +package com.seaofnodes.simple.transpile; + +import com.sun.source.util.JavacTask; + +import javax.tools.ToolProvider; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; + +import static java.util.Comparator.comparing; + +public class JUnitParser { + + /** + * @param simpleRepo The path to the project root + * @return A sorted map from chapter name to a sorted list of parsed test classes. + */ + public static TreeMap<String, List<TestClass>> parseRepository(Path simpleRepo) throws IOException { + var chapters = new TreeMap<String, List<TestClass>>(); + var inputs = simpleRepo.toFile().listFiles(File::isDirectory); + if (inputs == null) + throw new IOException("failed to read Simple project root"); + Arrays.sort(inputs); + for (File file : inputs) { + var chapter = file.getName(); + if (chapter.matches("chapter\\d+")) { + chapters.put(chapter, parseChapter(file.toPath())); + } + } + return chapters; + } + + /** + * @param input The path to a chapter in the repository. + * @return a sorted list of parsed test classes + */ + public static List<TestClass> parseChapter(Path input) throws IOException { + var tests = new ArrayList<Path>(); + Files.walkFileTree(input, Set.of(), 20, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.getFileName().toString().matches("Chapter\\d+Test\\.java")) { + tests.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + + var compiler = ToolProvider.getSystemJavaCompiler(); + var compilationUnits = compiler + .getStandardFileManager(null, null, null) + .getJavaFileObjectsFromPaths(tests); + + var task = (JavacTask) compiler.getTask(null, null, null, null, null, compilationUnits); + + var visitor = new AstVisitor(); + for (var tree : task.parse()) { + try { tree.accept(visitor, null); } + catch( IllegalArgumentException iae ) {} + } + + var testClasses = new ArrayList<>(visitor.results.values()); + testClasses.sort(comparing(TestClass::name)); + return testClasses; + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java new file mode 100644 index 00000000..5bb5609c --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java @@ -0,0 +1,6 @@ +package com.seaofnodes.simple.transpile; + +import java.util.ArrayList; + +public record TestClass(String name, ArrayList<TestMethod> methods) { +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java new file mode 100644 index 00000000..d0c6cfa8 --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java @@ -0,0 +1,39 @@ +package com.seaofnodes.simple.transpile; + +import java.util.ArrayList; + +public class TestMethod { + public String name; + public String parserInput; + public Arg parserArg; + // if this is set you can ignore the rest + public String parseErrorMessage; + public boolean disablePeephole; + public boolean showAfterParse; + public boolean iterate; + public boolean showAfterIterate; + public boolean irPrinter; + public boolean irPrinterLLVM; + public String assertStopEquals; + public boolean assertStopRetCtrlIsProj; + public boolean assertStopRetCtrlIsCProj; + public boolean assertStopRetCtrlIsRegion; + public final ArrayList<Evaluation> evaluations = new ArrayList<>(); + + public sealed interface Arg { + record IntBot() implements Arg { + } + + record IntTop() implements Arg { + } + + record IntConstant(long value) implements Arg { + } + } + + /** + * result is a literal value + **/ + public record Evaluation(Object result, long parameter) { + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java new file mode 100644 index 00000000..115a075a --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java @@ -0,0 +1,171 @@ +package com.seaofnodes.simple.transpile.printers; + +import com.seaofnodes.simple.transpile.JUnitParser; +import com.seaofnodes.simple.transpile.TestClass; +import com.seaofnodes.simple.transpile.TestMethod; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +public class RustPrinter { + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.out.println("Usage: <path-to-simple-repo> <output-dir>"); + System.exit(1); + } + var printer = new RustPrinter(); + var output = Path.of(args[1]); + + for (var e : JUnitParser.parseRepository(Path.of(args[0])).entrySet()) { + printer.printToFile(e.getKey(), e.getValue(), output); + } + } + + private void printToFile(String chapter, List<TestClass> testClasses, Path outputDirectory) throws IOException, InterruptedException { + var output = outputDirectory.resolve(chapter); + Files.createDirectories(output); + for (var testClass : testClasses) { + var outputFile = output.resolve(testClass.name().replace("Test", "").toLowerCase() + ".rs"); + System.out.println(outputFile); + Files.write(outputFile, printAndRustfmt(testClass.methods())); + } + } + + private byte[] printAndRustfmt(ArrayList<TestMethod> tests) throws IOException, InterruptedException { + var p = Runtime.getRuntime().exec(new String[]{"rustfmt"}); + try (var print = new PrintStream(new BufferedOutputStream(p.getOutputStream()))) { + printFile(tests, print); + } + p.waitFor(); + + var errors = new String(p.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); + if (!errors.isEmpty() || p.exitValue() != 0) { + throw new RuntimeException("rustfmt exit=" + p.exitValue() + " stderr=" + errors); + } + + return p.getInputStream().readAllBytes(); + } + + private void printFile(ArrayList<TestMethod> tests, PrintStream out) { + out.println(""" + use crate::datastructures::arena::DroplessArena; + use crate::sea_of_nodes::parser::Parser; + use crate::sea_of_nodes::types::Types;"""); + + if (tests.stream().anyMatch(t -> t.assertStopRetCtrlIsProj || t.assertStopRetCtrlIsCProj || t.assertStopRetCtrlIsRegion)) { + out.println("use crate::sea_of_nodes::nodes::Op;"); + } + if (tests.stream().anyMatch(t -> t.parseErrorMessage != null && !t.iterate)) { + out.println("use crate::sea_of_nodes::tests::test_error;"); + } + if (tests.stream().anyMatch(t -> t.parseErrorMessage != null && t.iterate)) { + out.println("use crate::sea_of_nodes::tests::test_error_iterate;"); + } + if (tests.stream().anyMatch(t -> t.irPrinter && !t.irPrinterLLVM)) { + out.println("use crate::sea_of_nodes::ir_printer::pretty_print;"); + } + if (tests.stream().anyMatch(t -> t.irPrinterLLVM)) { + out.println("use crate::sea_of_nodes::ir_printer::pretty_print_llvm;"); + } + if (tests.stream().anyMatch(t -> !t.evaluations.isEmpty())) { + out.println("use crate::sea_of_nodes::tests::evaluator::{evaluate, Object};"); + } + + for (TestMethod test : tests) { + printTest(test, out); + } + } + + private void printTest(TestMethod test, PrintStream out) { + out.println(); + out.println("#[test]"); + out.println("fn " + snakeCase(test.name) + "() {"); + if (test.parseErrorMessage != null) { + if (test.iterate) { + out.println("test_error_iterate(" + string(test.parserInput) + ", " + string(test.parseErrorMessage) + ");"); + } else { + out.println("test_error(" + string(test.parserInput) + ", " + string(test.parseErrorMessage) + ");"); + } + } else if (test.parserInput != null) { + out.println("let arena = DroplessArena::new();"); + out.println("let types = Types::new(&arena);"); + if (test.parserArg == null) { + out.println("let mut parser = Parser::new(" + string(test.parserInput) + ", &types);"); + } else { + out.printf("let mut parser = Parser::new_with_arg(%s, &types, %s);\n", string(test.parserInput), switch (test.parserArg) { + case TestMethod.Arg.IntBot ignored -> "types.ty_int_bot"; + case TestMethod.Arg.IntTop ignored -> "types.ty_int_top"; + case TestMethod.Arg.IntConstant c -> "types.get_int(" + c.value() + ")"; + }); + } + if (test.disablePeephole) { + out.println("parser.nodes.disable_peephole = true;"); + } + out.println("let stop = parser.parse().unwrap();"); + if (test.showAfterParse) { + out.println("parser.show_graph();"); + } + if (test.iterate) { + out.println("parser.iterate(stop);"); + out.println("parser.type_check(stop).unwrap();\n"); + } + if (test.showAfterIterate) { + out.println("parser.show_graph();"); + } + if (test.irPrinter) { + out.println("println!(\"{}\", pretty_print" + (test.irPrinterLLVM ? "_llvm" : "") + "(&parser.nodes, stop, 99));"); + } + if (test.assertStopEquals != null) { + out.println("assert_eq!(parser.print(stop), " + string(test.assertStopEquals) + ");"); + } + if (test.assertStopRetCtrlIsProj) { + out.println("assert!(matches!(parser.nodes.ret_ctrl(stop), Op::Proj(_)));"); + } + if (test.assertStopRetCtrlIsCProj) { + out.println("assert!(matches!(parser.nodes.ret_ctrl(stop), Op::CProj(_)));"); + } + if (test.assertStopRetCtrlIsRegion) { + out.println("assert!(matches!(parser.nodes.ret_ctrl(stop), Op::Region{..}));"); + } + for (var evaluation : test.evaluations) { + var result = switch (evaluation.result()) { + case null -> "Object::Null"; + case Integer l -> "Object::Long(" + l + ")"; + case Long l -> "Object::Long(" + l + ")"; + case Float d -> "Object::Double(" + d + ")"; + case Double d -> "Object::Double(" + d + ")"; + case String s -> string(s); + default -> + throw new RuntimeException("unexpected evaluation result: " + evaluation.result().getClass()); + }; + out.println("assert_eq!(" + result + ", evaluate(&parser.nodes, stop, Some(" + evaluation.parameter() + "), None).1);"); + } + } else { + out.println("todo!();"); + } + out.println("}"); + } + + private String string(String value) { + var escaped = value.replace("\\", "\\\\") + .replace("\r", "\\r") + .replace("\t", "\\t") + .replace("\"", "\\\""); + if (escaped.contains("\n")) { + return "\"\\\n" + escaped + "\""; + } else { + return "\"" + escaped + "\""; + } + } + + private String snakeCase(String identifier) { + return identifier.replaceAll("([A-Z]+)|([0-9]+)", "_$0").toLowerCase(); + } +} diff --git a/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java new file mode 100644 index 00000000..a2c3a782 --- /dev/null +++ b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java @@ -0,0 +1,26 @@ +package com.seaofnodes.simple.transpile; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TranspilerTest { + @Test + public void testParser() throws IOException { + var cwd = Path.of(".").toAbsolutePath().normalize(); + assertEquals("transpile-tests", cwd.getFileName().toString()); + var all = JUnitParser.parseRepository(cwd.getParent()); + assertTrue(all.size() >= 14); + + var chapter09 = all.get("chapter09"); + assertEquals(9, chapter09.size()); + + var class6 = chapter09.get(5); + assertEquals("Chapter06Test", class6.name()); + assertEquals(1, class6.methods().stream().filter(m -> m.name.equals("testPeepholeReturn")).count()); + } +} |
