diff options
| -rw-r--r-- | chapter32/Makefile | 31 | ||||
| -rw-r--r-- | chapter32/README.md | 775 | ||||
| -rw-r--r-- | chapter32/src/test/java/com/seaofnodes/simple/AssemblyDumpTest.java | 28 | ||||
| -rw-r--r-- | chapter32/src/test/java/com/seaofnodes/simple/ModTest.java | 27 | ||||
| -rw-r--r-- | chapter32/src/test/java/com/seaofnodes/simple/StringFormatTest.java | 113 | ||||
| -rw-r--r-- | chapter32/src/test/java/com/seaofnodes/simple/TestWorkspace.java | 2 | ||||
| -rw-r--r-- | chapter32/src/test/java/com/seaofnodes/simple/TypeTest.java | 81 | ||||
| -rw-r--r-- | skills/chapter25-codex-notes.md | 11 |
8 files changed, 394 insertions, 674 deletions
diff --git a/chapter32/Makefile b/chapter32/Makefile index af6da154..d91f59e0 100644 --- a/chapter32/Makefile +++ b/chapter32/Makefile @@ -30,13 +30,23 @@ test_cp := $(filter-out $(test_sys) ,$(test_cp4)) test_group0 := com.seaofnodes.simple.DrawTest test_group1 := com.seaofnodes.simple.GraphicsTest test_group2 := $(addprefix com.seaofnodes.simple.,TimeTest MemoryEncodingTest ChainedComparisonTest ParserTest Chapter28bTest) -test_group3 := $(addprefix com.seaofnodes.simple.,StringFormatTest FinalConstructionTest PartialReadTest EscapeFlowTest CallEndTest BulkMemPhiTest) -test_group4 := $(addprefix com.seaofnodes.simple.,BreakoutTest ArithNodeTest PhaseTest MethodAccessTest Int32Test MinusNodeTest ConstructorCompletionTest) -test_group5 := $(addprefix com.seaofnodes.simple.,TerminalInputTest ParserDiagnosticTest StaticDataTest ArrayInitTest MethodFieldsTest ConstantArrayTest BoundaryTest IntegerLiteralTest ConstructionRequirementsTest) -test_group6 := $(addprefix com.seaofnodes.simple.,AssemblyDumpTest AudioTest MathTest TicTacToeTest SelectiveLoadingTest FloatComparisonNativeTest SchedulingTest SeparateCompilationTest ConstructionGroupTest CompoundAssignmentTest) +test_group3 := $(addprefix com.seaofnodes.simple.,StringFormatTest StaticDataTest PhaseTest FinalConstructionTest PartialReadTest EscapeFlowTest CallEndTest BulkMemPhiTest) +test_group4 := $(addprefix com.seaofnodes.simple.,BreakoutTest ConvertNodeTest ArithNodeTest MethodAccessTest Int32Test MinusNodeTest ConstructorCompletionTest) +test_group5 := $(addprefix com.seaofnodes.simple.,TerminalInputTest ParserDiagnosticTest ArrayInitTest MethodFieldsTest ConstantArrayTest BoundaryTest IntegerLiteralTest ConstructionRequirementsTest) +test_group6 := $(addprefix com.seaofnodes.simple.,AssemblyDumpTest AudioTest MathTest TicTacToeTest ModTest SelectiveLoadingTest FloatComparisonNativeTest SchedulingTest SeparateCompilationTest ConstructionGroupTest CompoundAssignmentTest) test_group7 := $(filter-out $(test_group0) $(test_group1) $(test_group2) $(test_group3) $(test_group4) $(test_group5) $(test_group6),$(test_cp)) test_groups := $(addprefix tests_group,0 1 2 3 4 5 6 7) +# Fast compiler checks retain the full lattice, chapter/backend regressions, +# fuzzer regression seeds, and focused compiler tests. Library/application and +# CLI integration remains in `tests`. New compiler tests join sanity group 2. +sanity_omit := $(addprefix com.seaofnodes.simple.,DrawTest GraphicsTest TimeTest StringFormatTest BreakoutTest TerminalInputTest AudioTest MathTest TicTacToeTest StreamIOTest RuntimeArgsTest NativeExecutionTest ParserDiagnosticTest) +sanity_cp := $(filter-out $(sanity_omit),$(test_cp)) +sanity_group0 := $(addprefix com.seaofnodes.simple.,ClassInitDependenciesTest ConvertNodeTest ArithNodeTest MemoryEncodingTest ModTest IntrinsicTest AssemblyDumpTest) +sanity_group1 := $(addprefix com.seaofnodes.simple.,StaticDataTest FloatComparisonNativeTest ArrayInitTest ConstantArrayTest BoundaryTest Int32Test ReceiverTest GlobalAssignmentTest IncrementalBuildTest SeparateCompilationTest) +sanity_group2 := $(filter-out $(sanity_group0) $(sanity_group1),$(sanity_cp)) +sanity_groups := $(addprefix sanity_group,0 1 2) + # Size each test JVM for a share of the machine, instead of giving every JVM # JIT and GC pools sized for all cores. Absolute paths also serve child JVMs # launched from a group's private working directory (native Java on Windows). @@ -158,6 +168,15 @@ $(test_groups): tests_group%: $(test_classes) $(TEST_SYS) $(RELEASE_SYS) $(RELEA @echo "testing $@: $(test_group$*)" @cd build/tests/$@ && $(TEST_JVM) org.junit.runner.JUnitCore $(test_group$*) +.PHONY: sanity $(sanity_groups) +$(sanity_groups): sanity_group%: $(test_classes) $(TEST_SYS) $(RELEASE_SYS) $(RELEASE_CRT) + @$(TEST_JVM) com.seaofnodes.simple.TestWorkspace $@ $(CPU_ABI) + @echo "testing $@: $(sanity_group$*)" + @cd build/tests/$@ && $(TEST_JVM) org.junit.runner.JUnitCore $(sanity_group$*) + +sanity: $(sanity_groups) tests_raw0 tests_raw1 fuzzer tests_lattice0 tests_lattice1 tests_lattice2 tests_lattice3 + @echo "All sanity checks passed" + # One test method (or the whole class when METHOD is omitted). CLASS ?= Chapter25Test METHOD ?= @@ -165,7 +184,9 @@ METHOD ?= test: $(main_classes) $(test_classes) $(RELEASE_SYS) $(RELEASE_CRT) @$(TEST_JVM) com.seaofnodes.simple.TestRunner "$(CLASS)" "$(METHOD)" -tests: $(test_groups) tests_sys fuzzer tests_alone tests_raw1 tests_raw0 +# At -j8, start the remaining chapter cohorts before the short lattice parts +# so their native links overlap the earlier groups instead of forming a tail. +tests: $(test_groups) tests_sys fuzzer tests_raw1 tests_raw0 tests_alone @echo "All tests passed" diff --git a/chapter32/README.md b/chapter32/README.md index 7a814966..467b1792 100644 --- a/chapter32/README.md +++ b/chapter32/README.md @@ -4,598 +4,209 @@ English | [日本語](README.ja.md) [Previous: Chapter 31 — Software Drawing, Time, and Graphical Games](../chapter31/README.md) -Chapter 31 completes the graphics, games, and library work. This chapter keeps -that language and those programs, then investigates the cost of compiling and -testing them. Larger programs have made repeated graph walks, unnecessary -worklist visits, and uneven test parallelism visible in the test runtime. - -The starting changes address four costs: test JVM scheduling, scratch-state -cleanup in type queries, expensive verification, and overly broad optimizer -notifications. Each change keeps the existing native and evaluator coverage. - -## Measure the work before changing it - -A test suite includes Java startup, JIT compilation, assertions, compiler -passes, native compiler/linker subprocesses, and execution of generated code. -Timing the suite alone does not identify which part is growing. Likewise, -adding the timings of parallel JVMs does not give elapsed time. - -The investigation used JFR and temporary counters around compiler phases, -`_iter.iterate()`, expensive checks, and native subprocesses. Counters separate -worklist peepholes from the extra peepholes performed by verification. A -separate sample records the first accepted enqueue and follows it until the -node is popped, relating a null result to the code that scheduled the visit. -Duplicate pushes while a node is already queued are coalesced by the worklist. - -Use identity maps for bookkeeping keyed by Nodes. `Node.hashCode()` participates -in GVN locking; an ordinary HashMap used by a profiler can change compiler state. -The investigation helpers stay outside the production compiler. - -## Give independent tests independent JVMs - -The older Makefile ran chapter cohorts in parallel, but put the remaining tests -into one JVM. That last process became the long tail of `make -j 8 tests`. -The [Makefile](Makefile) divides those tests into eight measured groups, keeping -each class and its fixtures together. New test classes join the last group -automatically, so the grouping cannot silently omit them. - -Each JVM defaults to `-XX:ActiveProcessorCount=2`. This sizes its JIT and GC -thread pools for a share of the machine instead of letting every concurrent -JVM size itself for all cores. The limit does not pin the process to CPUs. - -Native tests create objects and executables with common filenames. -[TestWorkspace](src/test/java/com/seaofnodes/simple/TestWorkspace.java) gives -each group a working directory under `build/tests`, refreshing its fixtures -and native libraries before the run. Classes and JARs are shared through an -absolute classpath. The groups can then run without overwriting each other's -generated programs. - -The exhaustive [lattice test](src/test/java/com/seaofnodes/simple/TypeTest.java) -partitions its outer type loop across four JVMs. Together they check every -pair and triple. Separate processes also isolate the mutable type interning -and traversal state. A direct JUnit or IDE invocation still runs the complete -lattice test in one JVM; part zero of the Make run also runs the other type tests. - -## Shallow type queries need no recursive scratch state - -[Type.isConstant and Type.isFinal](src/main/java/com/seaofnodes/simple/type/Type.java) -used to wrap their queries in `recurOpen`/`recurClose`. That made sense when -constant detection could follow pointers into cyclic structures. The current -rules stop at the pointer: constantness tests its identity, and finality tests -its access qualifier. Struct queries inspect their fields without following -pointer referents. - -These queries therefore do not need to clear the shared traversal map, mark -visited structs, or treat a repeated visit as success. Removing that bookkeeping -keeps the existing answers and avoids repeated scratch-state cleanup. Recursive -operations that actually traverse or construct cyclic types retain their own -recursion machinery. Type tests cover repeated fields, cyclic singleton -pointers, and finality qualifiers. - -## Sample expensive verification - -The optimizer checks a strong invariant: a live node is either on the worklist -or cannot make progress. Testing it walks the entire graph and tries peepholes -again. `schedulableUses` also walks the graph to check that inputs and uses admit -a legal schedule. Running such checks too often can dominate the transformation -being checked. - -[CodeGen](src/main/java/com/seaofnodes/simple/codegen/CodeGen.java) now defaults -`simple.assert.expensive` to log 10 rather than 8, sampling one eligible step in -1,024. [IterPeeps](src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java) -checks scheduling at the end of the completed fixed point instead of after -every intermediate inlining batch. Entry, periodic, and phase checks remain. -Ordinary assertions remain enabled in the test harness. - -The verification-only change left the measured worklist visits identical at -4,463,457. Assertion-only peepholes fell from 5,295,983 to 1,910,132, and measured -checking time inside `iterate()` fell from 12.36 to 4.08 seconds. - -## Wake nodes for changes they actually read - -`listener.addDep(source)` puts the listener on the source's dependency list. -Later, `source.moveDepsToWorklist()` schedules those listeners. Notifications -can follow type changes, edge edits, or use-count changes. An unnecessary -subscription can therefore cause many visits even if the source's type never -changes. - -### Outside state is needed only for opaque arguments - -[SummaryNode](src/main/java/com/seaofnodes/simple/node/SummaryNode.java) and -[CallNode](src/main/java/com/seaofnodes/simple/node/CallNode.java) requested -`Start.outside` unconditionally while computing exposed escapes. That request -registers a dependency on Start. Deleting constants changes Start's users and -can wake every registered summary. - -But `TypeMem.escapesFrom(argument, outside)` consults `outside` only for an -opaque argument. The callers now use that same condition before requesting -outside state. Dependencies on argument types remain: if an argument later -refines to an opaque type, it wakes the reader, which installs the needed Start -dependency. The return-state dependencies of external CallEnds remain intact. - -### Removing a constant argument should not wake unrelated callers - -[RegionNode.removeDeadPath](src/main/java/com/seaofnodes/simple/node/RegionNode.java) -used to schedule every user of each removed Phi input. During inlining, common -constants can be shared by parameters throughout the program. Removing one -argument caused a broadcast to many unrelated functions. - -For a constant input, the code now schedules the changed Phi. Nonconstant -inputs retain the broader notification, and the Phi's outputs still receive -their notification. - -Stronger verification exposed a dependency on the old broadcast: -`CallNode.unlink` removed parameter inputs without scheduling the edited Parms. -Those nodes need another visit even when their types stay unchanged, because -editing their inputs removes them from GVN. Unlink now queues the parameters it -actually edits. A focused regression checks this independently of later cloning. - -### Results and remaining work - -With verification held at the new setting, the two filters produced: - -| Worklist peepholes | Before | After | -|---|---:|---:| -| SummaryNode | 455,186 | 50,623 | -| ParmNode | 700,827 | 373,604 | -| All nodes | 4,463,457 | 3,695,609 | -| Null returns | 4,140,886 | 3,371,682 | -| Null percentage | 92.8% | 91.2% | - -That is 17.2% fewer live worklist visits. Measured `iterate()` wall time fell -from 14.24 to 11.97 seconds; the serial profiled suite fell from 98.53 to 93.37 -seconds. These are individual Windows/JDK 21 runs with JFR and enqueue sampling, -not portable performance guarantees. The baseline had 792 test methods; the -final run included two additional regressions. The parallel uninstrumented -Make run took 28.8 seconds and passed 797 executions, including the additional -lattice partitions. The full suite also passed at log 8, with focused tests -at log 1. These measurements were taken before creating the chapter snapshot. - -A null return does not always mean wasted work: `peepholeOpt` can reinsert an -unlocked node into GVN and still return null. Some null-returning nodes are then -killed as unused. Already-dead pops are counted separately from this table. - -Dependency notifications accounted for only 28% of sampled baseline null-return -visits. Most came from direct scheduling. The largest remaining candidates are -MemPhi input requeues during Lower and IfNode dominator dependencies. The latter -also wake on use-count changes. Any narrower filter must preserve the topology -and use-count changes that actually enable a rewrite; deleting subscriptions -just because a sample showed no progress is not enough. - -### MemPhi and IfNode follow-up - -The next trace records the notifying node as well as the enqueue stack. -BulkMemPhi splitting and MemMerge cleanup repeatedly queue precise memory Phis -as inputs of successful peepholes. Removing memory Phis also changes a Region's -users, waking IfNodes which subscribed while walking the dominator chain. -Unlinking callers produces similar notifications from CallNodes. - -Two small filters are retained. A type-only update of a user no longer queues -an already-used, stable MemPhi input. New or edited inputs and first-use Phis -remain eligible; replacement and edge-edit notifications retain their existing -behavior. `IfNode.idealize` searches dominators only when both branches remain -possible. Known branches already fold through CProj and need no new distant -subscriptions. - -On the same 794-method profiling sweep, MemPhi visits fell from 474,510 to -469,866, IfNode visits from 254,787 to 249,609, and total visits from 3,695,609 -to 3,661,560. The total reduction is 0.9%; whole-suite elapsed times of 91.88 -and 91.64 seconds do not establish a meaningful timing improvement. Existing -spill expectations still pass. One graph expectation loses a redundant integer -narrowing; both branches also have execution checks. - -A broader experiment separated dependency use-count notifications from type -and input-edge notifications, and filtered more MemPhi use-count revisits. -That version changed optimizer ordering enough to increase the Sieve spill -score from 216 to 343 and the forward-field example from 23 to 31. It is not -enabled: reducing compiler work is not a reason to silently raise code-quality -expectations. The broader filter also requires notifying a node's dependents -when `delDef` changes its inputs, rather than redundantly notifying the old -input's use-count listeners. - -In the 302-method non-chapter group, that isolated experiment reduced MemPhi -visits from 367,387 to 205,245 and IfNode visits from 144,652 to 59,185. Total -visits fell 9.7%, but elapsed time was essentially unchanged at 57.48 versus -57.75 seconds. Fewer visits alone do not demonstrate a runtime improvement. - -These measurements precede the verifier fixes below. - -### Every-step verifier fixes - -Checking every progress step exposed two bugs, both reproducible before the -MemPhi/IfNode filters: - -- `Chapter05Test.testMerge4` fails at iteration 38: replacing `Phi#44` with - `arg` changes `If#52`'s predicate and enables `Phi#75`'s guard-merge rewrite. - `matchingGuardMerge` had no dependency on that predicate comparison. It now - subscribes to the If for direct input edits and to its predicate for replacement - through `subsume`. Tests cover both notification paths. -- `Chapter15Test.testArrayLoadDominatingControl` fails at iteration 30, when a - bulk memory Phi splits into a MemMerge. The verifier treated the consuming - load's earlier range-check control as its execution block. A memory operation's - control is only a lower bound: its data inputs can force it later. The check - now includes those inputs, matching early scheduling. Regression tests accept - this legal placement and still reject a load used before its memory is ready. - -After these fixes, all 802 suite executions pass at log 8; 444 chapter tests and -34 focused worklist, scheduling, and memory tests pass at log 1. Existing spill -expectations are unchanged. - -### Dense MemMerge inputs - -`MemMergeNode` now keeps precise alias slices in a dense input list, with an -`IntHashMap` from alias to input index and a reverse array for iteration. -Alias order is preserved. Missing aliases use input 1, the bulk default. -Deletion compacts the list and rebuilds the small map; graph copies own their -maps. Bulk-Phi splitting and object loading use alias lookup rather than -assuming an alias is an input index. The existing sparse object encoding remains -compatible with the baseline compiler. - -This is deliberately a storage experiment. `_aliasLimit` retains the previous -sparse extent for encoding and fold eligibility. The old `allDefault` test -counts null holes as non-defaults; simply folding whenever the dense list is -empty introduces additional optimizations. That separate trial altered code -quality: graphical Tic-tac-toe's weighted spill score rose from 1085 to 1240 -with its rebuilt library. The retained version preserves the old folding -behavior and matches baseline peephole counts, code sizes, and spill counts on -all measured workloads. - -Against commit `8b80d94c`, the final interleaved comparison measured: - -| Workload | Compile time change | Compiler-thread allocation change | -| --- | ---: | ---: | -| sys library | -5.8% | -3.7% | -| Clock | -2.1% | -10.8% | -| Breakout | -2.9% | -6.4% | -| Graphical tic-tac-toe | -0.5% | -5.3% | -| HelloWorld | -0.6% | -4.5% | - -The timing improvement is modest; sub-percent differences are within noise. -The structural reduction is much larger. After Lower, sys's live MemMerge input -slots fall from 54,388 to 2,462, Clock's from 54,255 to 2,237, and Breakout's from -61,286 to 2,592. These are input-slot counts, not total heap measurements; the -maps and reverse arrays have their own cost. - -Measurements use Java 21, assertions at log10, two active processors, and a -256 MiB initial / 2 GiB maximum heap. The compiler runs through object export, -without gcc, linking, or execution. Isolated compiler class loaders alternate -in one JVM; six measured rounds follow three warmups for each workload. Games -read the same baseline sys objects. Time changes are medians of paired ratios; -allocation uses ThreadMXBean. Earlier separate-process runs drifted enough that -the interleaved comparison is the basis for the table. - -The full suite passes at log8; 444 chapter tests and 18 memory/import tests pass -with every-step verification. Raw measurements, saved compiler classes, and -reproduction commands are in `build/memmerge-dense/`; verification logs are in -`build/test-profile/dense-final-*.log`. - -## Match interference scratch lifetime to function allocation +The games and libraries from chapter 31 made both compiling and testing slow +enough to interrupt development. This chapter makes room for future work: +faster feedback for Simple's developers, and faster compilation for its users. + +| What improved | Result | +| --- | --- | +| Repeated compilation | 29–79% less compile time for the large workloads below | +| Development testing | Roughly 30 seconds for a full run → 8.8 seconds for sanity checks; the full suite now takes 22.8 seconds | +| IR stored in ELF files | 21.9% smaller across 26 objects | + +The compiler figures isolate the allocator-scratch fix in repeated compiles. +The testing gain includes a deliberate coverage tradeoff. The sections below +explain what changed and what each measurement covers. + +## Faster development of Simple + +### Measure the work and the wait + +A test run starts JVMs, compiles Java methods, runs the compiler, checks +assertions, links native programs, and executes them. In one serial profiling +run, compiler phases took 29.7 seconds and 153 GCC calls took another 29.8, +mostly linking. Compiler optimization could address only part of the cost. + +Parallel testing has the "tallest pole" problem: we wait for the last worker. +The [Makefile](Makefile) now splits the former catch-all JVM into balanced +groups, with private native outputs. Each JVM sizes its JIT and GC pools for +two processors. Eight Make jobs beat both four and sixteen on this host. + +| Command | Coverage | Warm elapsed time | +| --- | --- | ---: | +| `make -j8 sanity` | Full lattice, chapter/backend tests, saved fuzzer regressions, focused compiler tests | 8.8 s | +| `make -j8 tests` | All of the above, plus remaining library, application, and CLI integration | 22.8 s | + +These runs use Windows, Java 21, and a 10-core/20-thread i9-10900K, with builds +already up to date. They execute 680 and 819 tests respectively, including the +lattice partitions. Assertions stay enabled. Use the full suite before finishing +a change, especially when working on libraries or interfaces omitted by `sanity`. + +Routine savings came from cheaper fixtures: loop over the division/remainder +matrix instead of generating 200 conditional checks; share native programs +between formatting cases; compile two small modules for an assembly-dump test +instead of all of `sys`. The full suite's GCC calls fell from 153 to 141. + +### Keep the full lattice, with fewer repetitions + +The [lattice test](src/test/java/com/seaofnodes/simple/TypeTest.java) stays in +the sanity run. Compiler transformations depend on its laws, so we reduced +repetition rather than sampling fewer types. + +First, format assertion messages only on failure. Doing that inside the cubic +loop reduced the standalone run from 10.6 to 4.5 seconds. Next, cache pairwise +meets and joins, computing both operand orders before checking commutativity. + +Associativity formerly visited all `n^3` ordered triples. Now it visits +`i <= j <= k`, including repeated operands: `n(n+1)(n+2)/6` iterations. Each +visit compares all six results: + +``` +(a & b) & c c & (a & b) +(a & c) & b b & (a & c) +(b & c) & a a & (b & c) +``` + +Once the original pairs commute, the six original associativity equalities +connect these results in a cycle. Comparing them to one common result checks +the same property. We must retain both operand orders involving intermediate +types: `a & b` may be outside the gathered set, so its commutativity has not +yet been established. + +All comparable-pair monotonicity checks remain. The complete unpartitioned +TypeTest now takes about 0.7 seconds. Make still partitions it across four +JVMs, each with private type-interning state. + +## Faster compilation for Simple users + +### Wake nodes when something relevant changes + +`listener.addDep(source)` asks to wake a node when the source changes. +Registering extra dependencies is easy; paying for them happens later. + +Tracing enqueues exposed two broad notifications. Calls and summaries read +`Start.outside` even when their arguments did not need it, subscribing them to +unrelated changes. They now read it only for opaque arguments. Removing a +constant Phi input also woke every user of that constant throughout the +program. It now queues the changed Phi. + +These filters cut worklist visits from 4.46 million to 3.70 million, or 17.2%. +Time inside `iterate()` fell from 14.24 to 11.97 seconds. Further MemPhi and +IfNode filters reduced visits slightly without a measurable timing gain. +Fewer visits alone are not proof of a faster compiler. + +Broad notifications can hide missing dependencies. Stronger verification found +nodes that had relied on accidental wakeups, including a Phi whose guard-merge +rewrite became possible after an If predicate changed. Checking every progress +step isolated the peephole that exposed the missing dependency. + +Normal runs sample expensive checks every 1,024 eligible steps, retaining entry +and completed-phase checks. Use `-Dsimple.assert.expensive=1` to check every +progress step while diagnosing a failure. Ordinary assertions remain enabled. + +### Give scratch data the lifetime of the work [RegAlloc](src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java) already -allocates one function at a time, retrying after spills before moving to the -next function. But [IFG](src/main/java/com/seaofnodes/simple/codegen/IFG.java)'s -static `BBOUTS` retained basic-block keys from every previous allocation, even -across compilations. Each interference build cleared all their live-out maps. -Repeated sys compiles added 2,558 keys apiece, making later compiles slower. - -Each function now creates an IFG instance. Its live-out maps, temporary live -set, and worklist survive that function's spill retries and become unreachable -when allocation returns. The shared interference bitsets still reuse capacity: -they contain live-range numbers, with no references to compiler graphs. - -Risky spill scoring also repeatedly walked the function's scheduled blocks to -compute the same area. Coloring now computes that area once, after coalescing, -and passes the integer to the scoring routines. A later spill retry recomputes -it because splitting can change the schedule. An assertion checks that the -live ranges being colored belong to the current function. - -Compared with `b8bcd9c0`, median compile times through object export were: - -| Workload | Before | After | RegAlloc before / after | -| --- | ---: | ---: | ---: | -| sys library | 1,290 ms | 913 ms | 494 / 91 ms | -| Clock | 537 ms | 279 ms | 274 / 27 ms | -| Breakout | 1,007 ms | 609 ms | 428 / 51 ms | -| Graphical tic-tac-toe | 1,362 ms | 292 ms | 1,113 / 40 ms | - -These are repeated compilations: six measured rounds after three warmups, -alternating isolated baseline and changed compiler class loaders. Each workload -starts with fresh loaders. The old implementation accumulates block keys even -during warmup, so these gains depend on compilation history; they are not -single-shot CLI startup measurements. Java 21 uses two active processors, -assertions at log10, and a 256 MiB initial / 2 GiB maximum heap. Both compilers -read the same sys objects; gcc, linking, and native execution are excluded. -HelloWorld remains about 69 ms, within measurement noise. - -Peephole counts, code sizes, and spill counts match in every benchmark pair. -The final 26 objects, including sys submodules, have identical `.text` bytes. -The full non-graphical suite passes all 805 test executions at log8, including -the native allocator checks. Benchmark sources, saved baseline classes, raw -timings, and follow-up JFR recordings are in `build/ifg-profile/`; the suite log -is `build/test-profile/ifg-full.log`. - -## Reduce allocation in load folding and packed writes - -The x86 [load-fold legality check](src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/x86_64_v2.java) -walks the other operand's inputs to ensure that combining it with a load cannot -move the read across a conflicting write. Its visited set now uses node IDs in -a local BitSet. The traversal, including control dependencies, is unchanged; -the check avoids growing arrays of identity-map keys and values. - -[BAOS](src/main/java/com/seaofnodes/simple/util/BAOS.java)'s packed-value -histogram was exploratory instrumentation for choosing smaller encodings. -Its map and updates are commented out, ready to restore for another packing -study. Normal compilation no longer boxes histogram keys and counts. Packed -bytes and their readers are unchanged. - -Against `6643e8fd`, an interleaved comparison measured sys compiler-thread -allocation falling from 119.6 to 69.4 MiB and median compile time from 1,017 to -942 ms. Breakout fell from 54.5 to 43.6 MiB and 652 to 617 ms. The smaller timing -differences were mixed: Clock measured 315 to 318 ms, graphical tic-tac-toe -292 to 291 ms, and HelloWorld 65 to 64 ms. All five allocate less. These use the -same three-warmup, six-measurement method and JVM settings as the IFG comparison. -All 26 resulting objects have byte-identical `.text` and `.simple` sections. -The Windows suite at log8 and the Linux suite at the default log10 each pass -all 805 test executions, including native scheduling and import coverage. - -## Measure serialized IR size before changing its packing - -The ELF `.simple` section stores reloadable ideal IR, declarations, and import -metadata. It can be much larger than the native code. For the same x86-64/win64 -objects, including all 22 sys modules: - -| Workload | Whole ELF bytes | `.simple` bytes | Share of ELF | -| --- | ---: | ---: | ---: | -| sys library, all modules | 597,399 | 469,881 | 78.7% | -| Clock | 64,317 | 43,854 | 68.2% | -| Breakout | 118,163 | 80,950 | 68.5% | -| Graphical tic-tac-toe | 64,605 | 44,353 | 68.7% | -| HelloWorld | 16,355 | 14,635 | 89.5% | - -Temporary byte-offset counters around [Serialize](src/main/java/com/seaofnodes/simple/codegen/Serialize.java)'s -existing write stages divide sys's payload into 67,844 bytes of strings and -unit metadata, 73,564 of identity mappings, 180,456 of types, 2,447 of function -summaries, and 145,482 of node records and input references, plus 88 header -bytes. HelloWorld's strings, mappings, and types alone take 14,161 of its 14,635 -bytes; its graph records and inputs take only 447 bytes. - -The broad metadata footprint warrants a separate investigation. The current -writer emits the full compilation's alias, function, and -return-PC mappings into each object, along with their names. Restricting them -to identities needed by that object's declarations, summaries, and graph may -save space, but requires coordinated remapping in the writer and reader. -Those measurements made no packing or format changes. Profiling helpers, -saved baseline classes, per-object sizes, byte breakdowns, and timings are in -`build/packing-profile/`. - -### Which packed fields have avoidable overhead? - -A deeper byte-count pass on `c21026c5` found three localized candidates: - -| Candidate | Modeled saving across sys | Clock | Breakout | -| --- | ---: | ---: | ---: | -| Unsigned varints for type-child IDs | 40,137 bytes | 2,757 | 3,561 | -| Sparse serialized MemMerge inputs | 34,280 bytes | 7,133 | 12,683 | -| Extend singleton function-ID encoding beyond ID 63 | 23,441 bytes | 1,140 | 1,409 | - -At that baseline, type-child IDs used `packed4`: IDs 128 through 16,383 took five bytes -where an unsigned varint would take two. [MemMerge](src/main/java/com/seaofnodes/simple/node/MemMergeNode.java) -was dense in memory, but its serialized representation retained the old alias -slot space; sys's MemMerge input data contained 36,138 zero bytes for default -slots. The sparse model keeps the first two inputs and encodes a non-default -count plus `(alias, node ID)` pairs, including the extra count/pair cost. -[TypeFunPtr](src/main/java/com/seaofnodes/simple/type/TypeFunPtr.java) had a -compact singleton form for non-null functions with fewer than six arguments, -but only below function ID 64. Larger singleton IDs became mostly-zero bitsets. - -These estimates concern disjoint fields and together amount to about 21% of -sys's `.simple` payload, 25% of Clock's, and 22% of Breakout's. The implementation -and actual measurements follow below. A generic signed-varint substitution is not equivalent: -zigzag encoding would make ordinary ASCII characters 64 through 127 larger. - -The metadata scan also explains HelloWorld's size. One constant pointing to -`class:sys` reaches 672 of its 700 serialized types, including unrelated -namespace members. Consequently, 363 of its 364 alias-map rows are referenced -by the current type representation. Dropping those rows alone is insufficient; -imported-type references or more selective namespace types need investigation. -Return-PC tables are a clearer candidate: only 214 of 7,788 RPC rows written -across sys objects are referenced by the serialized types, nodes, and summaries. - -Adding a general XInt dictionary is less promising: storing unique existing -encodings plus references is already slightly larger across sys, before table -headers. The singleton encoding addresses the common case more directly. -Detailed field counts, size models, and reproduction helpers are in -`build/simple-size/`. The instrumented exports preserve `.text` and `.simple` -bytes in all 27 measured objects; that investigation changed no format. - -## Pack the serialized IR more tightly - -[BAOS.packed4](src/main/java/com/seaofnodes/simple/util/BAOS.java) now uses -signed zigzag varints: -64 through 63 fit in one byte, -8192 through 8191 in -two, and the full signed int range takes at most five bytes. Each byte carries -seven payload bits and a continuation bit. Strings use a packed byte length -followed by raw UTF-8, keeping ordinary ASCII at one byte per character. - -MemMerge writes its control/default inputs and explicit `(alias, node ID)` -pairs, omitting default holes. It retains the old alias extent separately -because that value still affects folding eligibility. The reader remaps each -explicit alias into the importing compilation. Non-null singleton function -types with fewer than six arguments now use the compact form through function -ID 65535, with the matching `packed2` reader. - -Actual `.simple` sizes against `c21026c5`, including all 22 sys objects: - -| Workload | Before, bytes | After, bytes | Reduction | -| --- | ---: | ---: | ---: | -| sys library | 469,881 | 368,694 | 21.5% | -| Clock | 43,854 | 31,906 | 27.2% | -| Breakout | 80,950 | 62,139 | 23.2% | -| Graphical tic-tac-toe | 44,353 | 35,961 | 18.9% | -| HelloWorld | 14,635 | 11,523 | 21.3% | - -Across these 26 objects, the payload falls from 653,673 to 510,223 bytes, -saving 143,450 bytes (21.9%). Sys's type records and child IDs shrink from -180,456 to 116,948 bytes; node inputs shrink from 85,189 to 50,635. Together -those two chunks account for 96.9% of sys's saving. Identity mappings remain -73,564 bytes, leaving the metadata questions above for another pass. - -Compile timings are mixed. Baseline/new median milliseconds were sys -1,056/1,065, Clock 335/352, Breakout 689/674, graphical tic-tac-toe 282/283, -and HelloWorld 69/68. There is no consistent speed win in this run. The -comparison alternates two isolated compiler class loaders, with three warmups -and six measured rounds per workload, Java 21, two active processors, log10 -assertions, and a 256 MiB initial / 2 GiB maximum heap. Each compiler reads -its own freshly generated sys objects. Linking and execution are excluded. - -Every pair has matching peephole/no-progress counts, code sizes, and spill -metrics. All 26 `.text` sections are byte-identical. New tests cover signed -integer boundaries and overflow, Unicode strings, singleton function IDs, -sparse memory imports beyond alias 255, and rejection of the old format. -The Windows suite at log8 and Linux suite at log10 each pass 811 test executions. -Raw measurements, byte breakdowns, helpers, and saved baseline classes are in -`build/format-profile/`. - -The object header changes from `C0DE` to `C0D2`. Existing `.simple` objects -must be rebuilt; an old header produces an explicit rebuild error. For an -existing chapter32 build, `make clean` followed by `make -j 8 release` rebuilds -the generated artifacts. Rebuild any separately maintained Simple libraries too. - -## Receiver backport and spill investigation - -The shared class receiver is taught in -[chapter 28a](../chapter28a/README.md#globals-use-the-same-receiver-as-fields), -with imported summaries in [chapter 28b](../chapter28b/README.md). -Those fixes are carried through the later snapshots. Phi factoring after -strength reduction starts in -[chapter 5](../chapter05/README.md#factoring-after-strength-reduction), using -multiply-by-two there and shift-by-one where that representation is introduced. - -The receiver investigation also exposed a missed notification in -[RegionNode](src/main/java/com/seaofnodes/simple/node/RegionNode.java): flattening -regions adds Phi inputs, so their definitions must be queued too. A newly added -precise memory user can require its bulk-memory input to split immediately. -That prerequisite accompanies the receiver backport. - -The following measurements describe the chapter32 investigation before backporting. - -Validation: all 817 test executions pass on Windows with log8 verification and -Linux with the default log10. A focused 62-test run passes with log1, checking -every progress step. Native callbacks, imported receivers, FileIO, and ARM/RISC-V -execution remain covered. - -There is a compilation/code-size cost to review. The following Windows comparison -uses the preceding uncommitted assignment fix as its baseline, assertions enabled, -two JVM processors, seed 456, and each compiler's freshly built libraries. Times -are medians of four warm rounds after discarding the first round; this is a small -local sample, not a timing guarantee. Sys sizes sum its 22 object files. - -| Compile | Time, ms (before → after) | Peepholes (before → after) | `.text` bytes (before → after) | +allocated one function at a time, but its interference-graph helper retained +basic-block keys from every previous allocation. Each `sys` compile added +another 2,558 keys to clear on later runs. + +Each function now owns its [IFG](src/main/java/com/seaofnodes/simple/codegen/IFG.java) +scratch maps. They survive spill retries, then become unreachable when allocation +finishes. Spill scoring also computes the function's area once per coloring +attempt instead of walking the blocks for each candidate. + +| Program | Before | After | +| --- | ---: | ---: | +| `sys` library | 1,290 ms | 913 ms | +| Clock | 537 ms | 279 ms | +| Breakout | 1,007 ms | 609 ms | +| Graphical tic-tac-toe | 1,362 ms | 292 ms | + +These medians isolate that change: six measured compiles after three warmups, +alternating baseline and changed compilers, with the same library and assertions +enabled. They include object export but exclude linking. The old compiler +accumulated keys during warmup too; this measures repeated compilation, not +fresh command-line startup. Machine-code bytes matched in all 26 objects checked. + +### Remove empty space and unnecessary allocation + +[MemMergeNode](src/main/java/com/seaofnodes/simple/node/MemMergeNode.java) +used alias numbers as input indices, leaving mostly null slots supplied by +the default memory value. A dense list plus an alias-to-input map cut `sys`'s +live MemMerge input slots from 54,388 to 2,462. That change alone reduced its +compile time by 5.8%, while preserving folding behavior and generated code. + +Other savings were straightforward: remove recursive scratch-state setup from +the shallow `isConstant` and `isFinal` queries; use node-ID BitSets for load-fold +visits; stop collecting a packed-value histogram left over from profiling. +The latter two changes reduced `sys` compiler-thread allocation from 119.6 to +69.4 MiB and median compile time from 1,017 to 942 ms. These are separate +comparisons, not cumulative chapter-wide timings. + +### Make saved compiler state smaller + +ELF's `.simple` section stores reloadable IR, declarations, and import metadata. +It occupied 79% of the measured `sys` objects. Three encoding changes removed +avoidable overhead: gradual growth for packed integers, explicit MemMerge +alias/input pairs instead of default holes, and a wider compact encoding for +singleton function IDs. Strings retain raw UTF-8 bytes. + +| `.simple` payload | Before | After | Reduction | | --- | ---: | ---: | ---: | -| sys | 961 → 977 | 185,960 → 208,552 | 37,725 → 40,355 | -| Clock | 256 → 381 | 86,873 → 106,615 | 6,717 → 7,053 | -| Breakout | 553 → 593 | 158,712 → 174,500 | 14,868 → 15,031 | -| TicTacToe | 161 → 177 | 56,222 → 63,774 | 4,251 → 4,411 | - -The small String and HelloWorld examples also retain additional initialization -checks, and their spill expectations have been updated. The receiver-only change -exposed an existing ordering sensitivity in Sieve: across seeds 120–127, both -compilers produced seven 28-split/343-weight results and one 26-split/215-weight -result, with the smaller result moving from seed 126 to 121. - -The spill investigation separates compile time from generated moves: Clock's -weighted split score improves from 785 to 762. These scores count all retained -SplitNodes, including register copies, with weight `8^loopDepth`; they are not -counts of stack spills. Sieve's 215-to-343 change is two extra register copies -inside its depth-two loop. Early `p+p` strength reduction in -[AddNode](src/main/java/com/seaofnodes/simple/node/AddNode.java) turns the entry -value into `p<<1` before [PhiNode](src/main/java/com/seaofnodes/simple/node/PhiNode.java) -can factor `Phi(p+p,i+p)` into `Phi(p,i)+p`. - -Phi factoring now views `x<<1` as `x+x` when matching mixed Add/Shl arms. It uses -the existing factoring checks and trial graph, without expanding a shift into -an Add that would immediately strength-reduce again. Pure shift arms retain -ordinary shift factoring. Multiply-by-two already reduces to this shift, so it -is covered too. This is a local canonicalization: peeling one `x` from `k*x` for -arbitrary k would require a new `(k-1)*x` expression, with less clear cost and -rewrite-order benefits. The implementation stops at doubling. - -All eight Sieve seeds now produce 26 splits and score 215, with every-step -verification; the spill expectation is 215. Direct graph tests check factoring -before and after strength reduction, either arm order, and rejection of a -different shift count. The 819-test Windows sweep passes with log8 verification -(the Sieve group rerun after updating its expectation), and 64 focused tests -pass with log1. Native and emulator Sieve results are checked too. - -The largest individual sys increase in the receiver-only comparison was -`Canvas._circle`, from 455 to 517. -Allocation keeps the loop-invariant boolean in a general register and the pixel -pointer in an XMM register; recovering the pointer adds a transfer at each pixel -store. The baseline makes the opposite choice. Both ranges receive the default -score 1000 in [IFG.pickRiskyScore](src/main/java/com/seaofnodes/simple/codegen/IFG.java), -leaving their selection sensitive to ordering. An isolated trial giving SetX86 -candidates one extra priority point brings `_circle` to 454 and total sys to -8869, versus baseline 8719 and current 8932. Other sys functions' weighted scores -are unchanged by that trial. This identifies an allocator sensitivity; the -opcode-specific preference is diagnostic, not an adopted heuristic. - -With Phi factoring fixed, sys's weighted score is 8850 and `_circle` returns to -455 through another ordering change. Clock stays at 762, Breakout changes from -3387 to 3393, and TicTacToe stays at 1468. A bounded allocator follow-up tried -three cheap refinements of the default score: breaking ties toward fewer excess -neighbors (degree minus available colors), toward more excess neighbors, and a -bonus for cheap single-def/single-use splits based on loop depth. Sys scored -9039, 8844, and 8850 respectively; the three games were unchanged. The tiny -six-point gain does not justify adopting the degree tie-break on this evidence. -The production allocator remains unchanged; no additional graph walks or -searches were added. - -Backporting retains each chapter's allocator. Code-quality expectations are -therefore checked locally: chapter21's RISC-V Sieve score changes from 92 to -164, despite the chapter32 improvement above, and chapter31's forward-field -scanner test changes from 23 to 31 on x86-64/Win64. Execution checks remain in -place; these earlier snapshots do not inherit the later allocator experiments. -Chapter31 also groups `TimeTest` with its sys tests: its wide-library compile -passes in a fresh JVM, but following the large catch-all suite exhausted that -snapshot's existing 16-bit type IDs. This keeps the checks and the older type -implementation intact; it does not remove the compiler's type-ID limit. - -The saved baseline, comparison drivers, measurements, and verification logs are -under `build/receiver-profile/`. Proving initialization checks redundant is the -main follow-up suggested by these measurements. - -## Build and test - -From `chapter32`, using the existing offline dependency caches: +| `sys`, all 22 modules | 469,881 bytes | 368,694 bytes | 21.5% | +| Clock | 43,854 bytes | 31,906 bytes | 27.2% | +| Breakout | 80,950 bytes | 62,139 bytes | 23.2% | +| Graphical tic-tac-toe | 44,353 bytes | 35,961 bytes | 18.9% | +| HelloWorld | 14,635 bytes | 11,523 bytes | 21.3% | -```sh -make -j 8 release -make -j 8 tests_nographics -make test CLASS=TypeTest -``` +Across these 26 objects, the payload shrank by 143,450 bytes, or 21.9%, with +identical machine code. Compile timings showed no consistent improvement; +the demonstrated gain is smaller saved compiler state. + +## Keep the generated code good -On a fresh checkout, run `make libs` once while online to populate the caches. -The ordinary builds and tests do not download dependencies or open windows. +Changing worklist order can change rewrites and register allocation. Some +broader notification filters increased register copies, so we did not keep +them. Judge a compiler speedup by compile time, memory use, and generated code +together. -Override JVM sizing with `TEST_JAVA_OPTS`. For example: +Correctness fixes uncovered here belong in their teaching chapters: +[Phi factoring after strength reduction in chapter 5](../chapter05/README.md#factoring-after-strength-reduction), +and shared class receivers in [chapter 28a](../chapter28a/README.md#globals-use-the-same-receiver-as-fields) +and [chapter 28b](../chapter28b/README.md). + +## Build and use this chapter + +From `chapter32`, populate the dependency cache once with `make libs` if needed. +Ordinary builds use that cache and do not download dependencies. ```sh -make -j 4 tests_nographics TEST_JAVA_OPTS=-XX:ActiveProcessorCount=4 -make test CLASS=EscapeFlowTest "TEST_JAVA_OPTS=-XX:ActiveProcessorCount=2 -Dsimple.assert.expensive=1" +make -j8 release +make -j8 sanity +make -j8 tests ``` -Log 1 checks every progress step; log 8 restores the preceding sampling rate; -log 0 disables these expensive checks while retaining ordinary assertions. +For a fully headless full run, use `make -j8 tests_nographics`. It disables the +optional MiniFB and miniaudio backends. The packed IR header changes from `C0DE` +to `C0D2`; the reader rejects the older format. Run `make clean` before +`make -j8 release` to rebuild old objects, and rebuild any separately maintained +Simple libraries too. -## Language, libraries, and examples +Run one class, or enable every-step verification for a focused investigation: -The [grammar and parser map](docs/32-grammar.md) describes this snapshot. -Language and library additions are taught in [Chapter 31](../chapter31/README.md). -Their implementations and runnable examples are also present here: - -- [Examples and launch commands](docs/examples/README.md), including Clock, - Breakout, and graphical tic-tac-toe. -- [Drawing](src/main/smp/sys/draw/README.md), [window/input](src/main/smp/sys/gfx.md), - [audio](src/main/smp/sys/audio.md), and [time](src/main/smp/sys/time/README.md). -- [Math and intrinsics](../chapter31/README.md#math-support-and-intrinsics). +```sh +make test CLASS=TypeTest +make test CLASS=WorklistTest "TEST_JAVA_OPTS=-XX:ActiveProcessorCount=2 -Dsimple.assert.expensive=1" +``` -Future compiler and language proposals remain in the [roadmap](../docs/roadmap.md). +The [grammar and parser map](docs/32-grammar.md) describes this snapshot. +[Chapter 31](../chapter31/README.md) teaches its language and library additions; +the [examples](docs/examples/README.md) include launch commands for the games. +Library references live with [drawing](src/main/smp/sys/draw/README.md), +[window/input](src/main/smp/sys/gfx.md), [audio](src/main/smp/sys/audio.md), and +[time](src/main/smp/sys/time/README.md). Further projects belong in the +[roadmap](../docs/roadmap.md). + +Use the fast checks while editing. Run the full suite before finishing, and +turn up verification when a failure needs a closer look. diff --git a/chapter32/src/test/java/com/seaofnodes/simple/AssemblyDumpTest.java b/chapter32/src/test/java/com/seaofnodes/simple/AssemblyDumpTest.java index 0e4c230a..5399aecf 100644 --- a/chapter32/src/test/java/com/seaofnodes/simple/AssemblyDumpTest.java +++ b/chapter32/src/test/java/com/seaofnodes/simple/AssemblyDumpTest.java @@ -13,25 +13,39 @@ public class AssemblyDumpTest { @Test public void libraryUnitsAndGlyphs() throws Exception { Path root=Path.of("build/objs/assembly-dump"); Files.createDirectories(root); - Path build=Files.createTempDirectory(root,"sys-"); - CodeGen code=new CodeGen("src/main/smp",build.toString(),null,"sys",null,123,true,TypeInteger.BOT) + Path fixture=Files.createTempDirectory(root,"font-"); + Path source=Files.createDirectories(fixture.resolve("src")); + Path build=Files.createDirectories(fixture.resolve("obj")); + Files.createDirectories(source.resolve("Atlas")); + // Use the real font, but only two tiny modules: drawing integration is + // covered by DrawTest. This test needs multiple units and constant data. + String canvasSource=Files.readString(Path.of("src/main/smp/sys/draw/Canvas.smp")); + int fontStart=canvasSource.indexOf("val _glyphs ="); + assertTrue("Font declaration",fontStart>=0); + int fontEnd=canvasSource.indexOf("};",fontStart); + assertTrue("Font initializer",fontEnd>=0); + Files.writeString(source.resolve("Atlas/Font.smp"),canvasSource.substring(fontStart,fontEnd+2)+ + "\nval glyph={int i -> _glyphs[i];};\n"); + Files.writeString(source.resolve("Atlas.smp"),"val glyph={int i -> Font.glyph(i);};\n"); + CodeGen code=new CodeGen(source.toString(),build.toString(),null,"Atlas",null,123,true,TypeInteger.BOT) .driver(CodeGen.Phase.Serialize,TestC.CPU_PORT,TestC.CALL_CONVENTION) .driver(CodeGen.Phase.Encoding,TestC.CPU_PORT,TestC.CALL_CONVENTION); var entry=code._encoding; String before=code.asm(); assertSame("Dumping another unit must not change the active encoding",entry,code._encoding); + assertEquals(2,code._compunits.size()); for( var unit : code._compunits.values() ) if( unit._src!=null ) { String heading="=== Compilation unit: "+unit._cname+" ==="; assertEquals("Exactly one heading for "+unit._cname,2,before.split(heading,-1).length); } - int start=before.indexOf("=== Compilation unit: sys.draw.Canvas ==="); + int start=before.indexOf("=== Compilation unit: Atlas.Font ==="); assertTrue(start>=0); int end=before.indexOf("=== Compilation unit:",start+1); String canvas=before.substring(start,end<0 ? before.length() : end); assertTrue(canvas.contains("Constant Pool")); - assertTrue(canvas.contains("sys.draw.Canvas._glyphs: []u16")); - TypeMemPtr ptr=(TypeMemPtr)code._compunits.get("sys/draw/Canvas")._clz.field("_glyphs")._t; + assertTrue(canvas.contains("Atlas.Font._glyphs: []u16")); + TypeMemPtr ptr=(TypeMemPtr)code._compunits.get("Atlas/Font")._clz.field("_glyphs")._t; TypeConAry glyphs=(TypeConAry)ptr._obj.field("[]")._t; assertEquals(96,glyphs.len()); BAOS bytes=new BAOS(); @@ -40,10 +54,10 @@ public class AssemblyDumpTest { canvas.contains("60000000"+HexFormat.of().formatHex(bytes.toByteArray()))); code.exportELF(false,false); com.seaofnodes.simple.codegen.PrintRegTestSupport.checkDataAliases( - build.resolve("sys/draw/Canvas.o"),196,"sys.draw.Canvas._glyphs"); + build.resolve("Atlas/Font.o"),196,"Atlas.Font._glyphs"); assertSame(entry,code._encoding); assertEquals("Export must not truncate the dump to its entry unit",before,code.asm()); - Files.writeString(build.resolve("sys.asm"),before); + Files.writeString(build.resolve("Atlas.asm"),before); } @Test public void combinedEncodingAndEarlierPhase() { diff --git a/chapter32/src/test/java/com/seaofnodes/simple/ModTest.java b/chapter32/src/test/java/com/seaofnodes/simple/ModTest.java index 7f79599d..c06e99f1 100644 --- a/chapter32/src/test/java/com/seaofnodes/simple/ModTest.java +++ b/chapter32/src/test/java/com/seaofnodes/simple/ModTest.java @@ -166,20 +166,31 @@ public class ModTest { return integerEdges(new long[]{Long.MIN_VALUE,Long.MIN_VALUE+1,-17,-5,-1,0,1,5,17,Long.MAX_VALUE}); } private static String integerEdges(long[] values) { + // Keep the full value matrix in data, rather than compiling a long + // chain of constant calls and conditional returns for every pair. StringBuilder source=new StringBuilder(""" val rem_noInline={ i64 x,i64 y -> x%y; }; val div_noInline={ i64 x,i64 y -> x/y; }; val main={ -> + val values=new i64[~]{ """); - int failure=1; - for( long x : values ) - for( long y : values ) { - source.append("if(rem_noInline(").append(literal(x)).append(',').append(literal(y)) - .append(")!=").append(literal(y==0 ? 0 : x%y)).append(") return ").append(failure++).append(';'); - source.append("if(div_noInline(").append(literal(x)).append(',').append(literal(y)) - .append(")!=").append(literal(y==0 ? 0 : x/y)).append(") return ").append(failure++).append(';'); + for(int i=0;i<values.length;i++) source.append(i==0 ? "" : ",").append(literal(values[i])); + source.append("}; val expectedRem=new i64[~]{"); + int n=0; + for(long x:values) for(long y:values) source.append(n++==0 ? "" : ",").append(literal(y==0 ? 0 : x%y)); + source.append("}; val expectedDiv=new i64[~]{"); + n=0; + for(long x:values) for(long y:values) source.append(n++==0 ? "" : ",").append(literal(y==0 ? 0 : x/y)); + return source.append(""" + }; + int index=0; + for(int i=0;i<values#;i++) for(int j=0;j<values#;j++) { + if(rem_noInline(values[i],values[j])!=expectedRem[index]) return index*2+1; + if(div_noInline(values[i],values[j])!=expectedDiv[index]) return index*2+2; + index++; } - return source.append("return 0; };").toString(); + return 0; }; + """).toString(); } @Test public void nativeFloatingEdges() throws Exception { double[][] cases={{Double.MAX_VALUE,0.1},{-Double.MAX_VALUE,0.1}, diff --git a/chapter32/src/test/java/com/seaofnodes/simple/StringFormatTest.java b/chapter32/src/test/java/com/seaofnodes/simple/StringFormatTest.java index 59b62709..d6783380 100644 --- a/chapter32/src/test/java/com/seaofnodes/simple/StringFormatTest.java +++ b/chapter32/src/test/java/com/seaofnodes/simple/StringFormatTest.java @@ -11,28 +11,88 @@ import org.junit.Test; import static org.junit.Assert.*; public class StringFormatTest extends TestC { - private static void prints(String body, String expected) throws Exception { - runSYS("val main={ -> "+body+" return 0; };", "stringFormat",expected,-1); + // Share a few native programs, retaining a separately reported assertion + // for each case. Blocks isolate locals; the delimiter preserves embedded + // NULs and newlines in the expected output. + private enum Format { + DefaultsAndNestedStrings("sys.io.p(\"CPU plays %(2+3). %(\"nested %(7)\") %(true) %(1.5)\\n\");", + "CPU plays 5. nested 7 1 1.5\n"), + Formats("sys.io.p(\"%+06d(17)|%#08x(42)|%-5.3s(\"abcdef\")|%.2f(1.25)|%c(65)|%b(false)\\n\");", + "+00017|0x00002a|abc |1.25|A|false\n"), + RemainderInInterpolation("int mod=17; sys.io.p(\"%(mod%5)|%02d(mod%5)|%%|\\%|%(mod %= 5)|%(mod)\");", + "2|02|%|%|2|2"), + IntegerLimits("i64 lo=-9223372036854775807-1; sys.io.p(\"%(lo)|%(9223372036854775807)|%u(-1)|%X(-1)\");", + "-9223372036854775808|9223372036854775807|18446744073709551615|FFFFFFFFFFFFFFFF"), + IntegerPaddingRules("sys.io.p(\"%#.0o(0)|%#.3o(1)|%#x(0)|%-08d(17)|%08.4d(-17)|%+d(0)|% d(0)\");", + "0|001|0|17 | -0017|+0| 0"), + BuilderNumberUsesFormatter("val b=new sys.aryu8(); b.num(-9223372036854775807-1).add('|').num(9223372036854775807); sys.io.p(sys.text.end(b));", + "-9223372036854775808|9223372036854775807"), + EvaluationOrder("int x=0; val s=\"%(x++)%(x++)%(x++)\"; sys.io.p(\"%(s):%(s#):%(x)|%%|\\%\");", + "012:3:3|%|%"), + EvaluationMutatesBytes("val a=new u8[1]; a[0]='A'; sys.io.p(\"%(a)%(a[0]='B')%(a)\");", "A66B"), + Converters("sys.io.p(sys.text.int(17)); sys.io.p(sys.text.char(65)); sys.io.p(sys.text.bool(true)); sys.io.p(sys.text.flt(1.5));", + "17Atrue1.5"), + FieldNamedInt("struct S { int int=7; }; sys.io.p(\"%((new S).int)\");", "7"), + FloatAndAlternateFormats("f32 x=1.25f; sys.io.p(\"%(x)|%f(x)|%.2e(x)|%.2G(x)|%#o(9)|% d(7)|%.0d(0)|%a(1.5)\");", + "1.25|1.250000|1.25e+00|1.2|011| 7||0x1.8p+0"), + BufferGrowth("sys.io.p(\"%0100d(17)|%.100f(1.0)\");", "0".repeat(98)+"17|1."+"0".repeat(100)), + BytesAndEmptyStrings("val empty=\"%(\"\")\"; val bytes=\"a\\0b\"; sys.io.p(\"%(empty#):%s(bytes):%.2s(bytes):%c(0)\");", + "0:a\0b:a\0:\0"), + Utf8String("val s=\"π\"; sys.io.p(\"%(s):%(s#):%(s[0]):π\");", "π:2:207:π"), + DeferredParsing(""" + struct S { u8[~] s; new S={ -> s="%("}) %(3)")"; }; }; + val s=new S; + sys.io.p(s.s); + for(int i=0; i<2; sys.io.p("%(i++) %("(")")) { } + """, "}) 30 (1 ("); + + final String body, expected; + Format(String body, String expected) { this.body=body; this.expected=expected; } + } + private static final String END="\n--end-format-case--\n"; + private static final int BATCH_SIZE=5; + private static final String[] OUTPUTS=new String[Format.values().length]; + + @org.junit.BeforeClass public static void resetNativeBatches() { + java.util.Arrays.fill(OUTPUTS,null); + } + + private static void prints(Format test) throws Exception { + int index=test.ordinal(); + if( OUTPUTS[index]==null ) { + Format[] cases=Format.values(); + int first=index/BATCH_SIZE*BATCH_SIZE, limit=Math.min(first+BATCH_SIZE,cases.length); + StringBuilder source=new StringBuilder("val main={ -> "); + for( int i=first; i<limit; i++ ) { + assertFalse("Reserved batch delimiter",cases[i].expected.contains(END)); + source.append('{').append(cases[i].body).append('}'); + source.append("sys.io.p(\"\\n--end-format-case--\\n\");"); + } + source.append("return 0; };"); + String exe=compile(source.toString(),"stringFormat"+first, + new Ary<>(new String[]{RELEASE_SYS_DIR}),CALL_CONVENTION,null,null,-1); + String[] outputs=exec(TEST_TIMEOUT_SECONDS,exe).split(java.util.regex.Pattern.quote(END),-1); + assertEquals("Each format case must complete",limit-first+1,outputs.length); + assertEquals("No output after final case", "",outputs[outputs.length-1]); + for( int i=first; i<limit; i++ ) OUTPUTS[i]=outputs[i-first]; + } + assertEquals(test.name(),test.expected,OUTPUTS[index]); } @Test public void testDefaultsAndNestedStrings() throws Exception { - prints("sys.io.p(\"CPU plays %(2+3). %(\"nested %(7)\") %(true) %(1.5)\\n\");", - "CPU plays 5. nested 7 1 1.5\n"); + prints(Format.DefaultsAndNestedStrings); } @Test public void testFormats() throws Exception { - prints("sys.io.p(\"%+06d(17)|%#08x(42)|%-5.3s(\"abcdef\")|%.2f(1.25)|%c(65)|%b(false)\\n\");", - "+00017|0x00002a|abc |1.25|A|false\n"); + prints(Format.Formats); } @Test public void testRemainderInInterpolation() throws Exception { - prints("int mod=17; sys.io.p(\"%(mod%5)|%02d(mod%5)|%%|\\%|%(mod %= 5)|%(mod)\");", - "2|02|%|%|2|2"); + prints(Format.RemainderInInterpolation); } @Test public void testIntegerLimits() throws Exception { - prints("i64 lo=-9223372036854775807-1; sys.io.p(\"%(lo)|%(9223372036854775807)|%u(-1)|%X(-1)\");", - "-9223372036854775808|9223372036854775807|18446744073709551615|FFFFFFFFFFFFFFFF"); + prints(Format.IntegerLimits); } @Test public void testSignedAndUnsignedRadices() throws Exception { @@ -49,13 +109,11 @@ public class StringFormatTest extends TestC { } @Test public void testIntegerPaddingRules() throws Exception { - prints("sys.io.p(\"%#.0o(0)|%#.3o(1)|%#x(0)|%-08d(17)|%08.4d(-17)|%+d(0)|% d(0)\");", - "0|001|0|17 | -0017|+0| 0"); + prints(Format.IntegerPaddingRules); } @Test public void testBuilderNumberUsesFormatter() throws Exception { - prints("val b=new sys.aryu8(); b.num(-9223372036854775807-1).add('|').num(9223372036854775807); sys.io.p(sys.text.end(b));", - "-9223372036854775808|9223372036854775807"); + prints(Format.BuilderNumberUsesFormatter); } @Test public void testFormattingInEvaluatorFromSource() { @@ -74,15 +132,13 @@ public class StringFormatTest extends TestC { } @Test public void testEvaluationOrderAndLength() throws Exception { - prints("int x=0; val s=\"%(x++)%(x++)%(x++)\"; sys.io.p(\"%(s):%(s#):%(x)|%%|\\%\");", - "012:3:3|%|%"); - prints("val a=new u8[1]; a[0]='A'; sys.io.p(\"%(a)%(a[0]='B')%(a)\");", "A66B"); + prints(Format.EvaluationOrder); + prints(Format.EvaluationMutatesBytes); } @Test public void testConverters() throws Exception { - prints("sys.io.p(sys.text.int(17)); sys.io.p(sys.text.char(65)); sys.io.p(sys.text.bool(true)); sys.io.p(sys.text.flt(1.5));", - "17Atrue1.5"); - prints("struct S { int int=7; }; sys.io.p(\"%((new S).int)\");", "7"); + prints(Format.Converters); + prints(Format.FieldNamedInt); } @Test public void testForwardReturnTypes() throws Exception { @@ -91,14 +147,13 @@ public class StringFormatTest extends TestC { } @Test public void testFloatAndAlternateFormats() throws Exception { - prints("f32 x=1.25f; sys.io.p(\"%(x)|%f(x)|%.2e(x)|%.2G(x)|%#o(9)|% d(7)|%.0d(0)|%a(1.5)\");", - "1.25|1.250000|1.25e+00|1.2|011| 7||0x1.8p+0"); + prints(Format.FloatAndAlternateFormats); runSYS("val format_noInline={ f32 x -> \"%(x)|%f(x)\"; }; val main={ -> sys.io.p(format_noInline(1.25f)); return 0; };", "stringFormatFloat","1.25|1.250000",-1); } @Test public void testBufferGrowth() throws Exception { - prints("sys.io.p(\"%0100d(17)|%.100f(1.0)\");", "0".repeat(98)+"17|1."+"0".repeat(100)); + prints(Format.BufferGrowth); } @Test public void testLiteralRemainsConstant() { @@ -107,18 +162,12 @@ public class StringFormatTest extends TestC { } @Test public void testBytesAndEmptyStrings() throws Exception { - prints("val empty=\"%(\"\")\"; val bytes=\"a\\0b\"; sys.io.p(\"%(empty#):%s(bytes):%.2s(bytes):%c(0)\");", - "0:a\0b:a\0:\0"); - prints("val s=\"π\"; sys.io.p(\"%(s):%(s#):%(s[0]):π\");", "π:2:207:π"); + prints(Format.BytesAndEmptyStrings); + prints(Format.Utf8String); } @Test public void testDeferredParsing() throws Exception { - prints(""" - struct S { u8[~] s; new S={ -> s="%("}) %(3)")"; }; }; - val s=new S; - sys.io.p(s.s); - for(int i=0; i<2; sys.io.p("%(i++) %("(")")) { } - """, "}) 30 (1 ("); + prints(Format.DeferredParsing); } @Test public void testImportedFormatter() throws Exception { diff --git a/chapter32/src/test/java/com/seaofnodes/simple/TestWorkspace.java b/chapter32/src/test/java/com/seaofnodes/simple/TestWorkspace.java index 30cff89f..d875c47a 100644 --- a/chapter32/src/test/java/com/seaofnodes/simple/TestWorkspace.java +++ b/chapter32/src/test/java/com/seaofnodes/simple/TestWorkspace.java @@ -7,7 +7,7 @@ import java.nio.file.attribute.BasicFileAttributes; /** Prepare a Make test group's working directory without sharing native outputs. */ public class TestWorkspace { public static void main(String[] args) throws IOException { - if( args.length!=2 || !args[0].matches("tests_group[0-7]") || !args[1].matches("[A-Za-z0-9_]+") ) + if( args.length!=2 || !args[0].matches("tests_group[0-7]|sanity_group[0-2]") || !args[1].matches("[A-Za-z0-9_]+") ) throw new IllegalArgumentException("Expected test group and CPU_ABI"); Path root = Path.of("").toAbsolutePath().normalize(); Path work = root.resolve("build/tests").resolve(args[0]).normalize(); diff --git a/chapter32/src/test/java/com/seaofnodes/simple/TypeTest.java b/chapter32/src/test/java/com/seaofnodes/simple/TypeTest.java index 8f212bda..adcc1b3a 100644 --- a/chapter32/src/test/java/com/seaofnodes/simple/TypeTest.java +++ b/chapter32/src/test/java/com/seaofnodes/simple/TypeTest.java @@ -276,74 +276,77 @@ public class TypeTest { public void testLatticeTheory() { Type[] ts = Type.gather(); // Make partitions the outer loop between independent JVMs. A direct - // JUnit/IDE invocation retains the full Cartesian-product coverage. + // JUnit/IDE invocation retains the full coverage. int parts = Integer.parseInt(System.getProperty("simple.lattice.parts","1")); int part = Integer.parseInt(System.getProperty("simple.lattice.part" ,"0")); Assert.assertTrue("Invalid lattice partition",parts > 0 && parts <= ts.length && part >= 0 && part < parts); - // Confirm commutative & complete - for( int i=part; i<ts.length; i+=parts ) { - Type t0 = ts[i]; - for( Type t1 : ts ) { - check_commute (t0,t1); - check_symmetric(t0,t1); + // Every triple reuses these pairs. Compute both operand orders rather + // than assuming commutativity in the table we are about to check. + Type[][] meets = new Type[ts.length][ts.length]; + Type[][] joins = new Type[ts.length][ts.length]; + for( int i=0; i<ts.length; i++ ) + for( int j=0; j<ts.length; j++ ) { + meets[i][j] = ts[i].meet(ts[j]); + joins[i][j] = ts[i].join(ts[j]); } - } - // Confirm associative + // Confirm commutative & complete, once per unordered pair. for( int i=part; i<ts.length; i+=parts ) { - Type t0 = ts[i]; - for( Type t1 : ts ) - for( Type t2 : ts ) - assoc(t0,t1,t2); + assertSame(ts[i],meets[i][i]); + for( int j=i; j<ts.length; j++ ) { + assertSame(meets[i][j],meets[j][i]); + check_symmetric(ts[i],ts[j],meets[i][j]); + } } + // Each unordered triple represents all six permutations. Check every + // intermediate result in both operand orders; a pair's meet need not + // itself be in ts, so its commutativity must not be assumed. + for( int i=part; i<ts.length; i+=parts ) + for( int j=i; j<ts.length; j++ ) + for( int k=j; k<ts.length; k++ ) + assoc(ts[i],ts[j],ts[k],meets[i][j],meets[i][k],meets[j][k]); + // Confirm symmetry. If A isa B, then A.join(C) isa B.join(C) for( int i=part; i<ts.length; i+=parts ) { Type t0 = ts[i]; - for( Type t1 : ts ) - if( t0.isa(t1) ) - for( Type t2 : ts ) { - Type t02 = t0.join(t2); - Type t12 = t1.join(t2); + for( int j=0; j<ts.length; j++ ) + if( meets[i][j]==ts[j] ) + for( int k=0; k<ts.length; k++ ) { + Type t02 = joins[i][k]; + Type t12 = joins[j][k]; Type mt = t02.meet(t12); - assertSame(t0+" isa "+t1+" joined "+t2, mt,t12); + if( mt!=t12 ) assertSame(t0+" isa "+ts[j]+" joined "+ts[k], mt,t12); } } } - // By design in meet, args are already flipped to order _type, which forces - // symmetry for things with badly ordered _type fields. The question is - // still interesting for other orders. - private static void check_commute( Type t0, Type t1 ) { - if( t0==t1 ) return; - if( t0.is_simple() && !t1.is_simple() ) return; // By design, flipped the only allowed order - Type mta = t0.meet(t1); - Type mtb = t1.meet(t0); // Reverse args and try again - assertSame(mta,mtb); - } - // A & B = MT // Expect: ~A & ~MT == ~A // Expect: ~B & ~MT == ~B private static void check_symmetric( Type t0, Type t1 ) { + check_symmetric(t0,t1,t0.meet(t1)); + } + + private static void check_symmetric( Type t0, Type t1, Type mt ) { if( t1==t0 ) return; - Type mt = t0.meet(t1); Type dm = mt.dual(); Type d0 = t0.dual(); Type d1 = t1.dual(); Type ta = dm.meet(d1); Type tb = dm.meet(d0); - assertSame(t0+" & "+t1+" mt="+mt, ta,d1); - assertSame(t0+" & "+t1+" mt="+mt, tb,d0); + if( ta!=d1 ) assertSame(t0+" & "+t1+" mt="+mt, ta,d1); + if( tb!=d0 ) assertSame(t0+" & "+t1+" mt="+mt, tb,d0); } - private static void assoc( Type t0, Type t1, Type t2 ) { - Type t01 = t0 .meet(t1 ); - Type t12 = t1 .meet(t2 ); - Type t01_2 = t01.meet(t2 ); - Type t0_12 = t0 .meet(t12); - assertSame(t0+" & "+t1+" & "+t2, t01_2,t0_12); + private static void assoc( Type a, Type b, Type c, Type ab, Type ac, Type bc ) { + // The six original associativity equalities connect these six values + // in a cycle. Equality to one common result checks the same property. + Type result = ab.meet(c); + if( result!=c.meet(ab) || result!=ac.meet(b) || result!=b.meet(ac) || + result!=bc.meet(a) || result!=a.meet(bc) ) + Assert.fail("Associativity: "+a+" & "+b+" & "+c); } // Test cyclic types and meets diff --git a/skills/chapter25-codex-notes.md b/skills/chapter25-codex-notes.md index 7cf22839..b32fdd15 100644 --- a/skills/chapter25-codex-notes.md +++ b/skills/chapter25-codex-notes.md @@ -35,6 +35,17 @@ callers promise no initialized fields. No new passes or backward demand walks. ## Chapter 32: restart guidance (2026-11-15) +Use `make -j8 tests` as the normal validation/timing command going forward. +The November 16 sweep found both `-j4` and `-j16` slower on this Windows host; +retain `ActiveProcessorCount=2`. Keep lattice checks in their own JVMs: combining +them after compiler-heavy tests can exhaust the existing 16-bit type IDs. +The full lattice is required even in fast checks. `make -j8 sanity` keeps its +four partitions plus chapter/backend and focused compiler regressions; full +`make -j8 tests` retains library/application/CLI integration. The lattice now +caches pairs and checks all six intermediate results per unordered triple; +do not reduce it to one associativity equality per unordered triple, which +would assume commutativity for intermediate types outside the gathered set. + On 2026-11-16, Cliff requested backporting the shared class receiver to chapter 28a, imported-summary handling to 28b, and both through the later snapshots. The chapter28a README now owns that explanation. Preserve 28a/28b's monolithic |
