SourcePartitionGuard.kt

    @JvmStatic
    fun main(args: Array<String>) {
        val verbose = args.contains("--verbose")
        val repo = repoRoot()
        val sourceRoot = File(repo, "src")
        if (!sourceRoot.isDirectory) {
            System.err.println("SourcePartitionGuard: no src/ under ${repo.absolutePath}")
            exitProcess(2)
        }

        val files = sourceRoot.walkTopDown()
            .filter { it.isFile && it.extension == "kt" }
            .filterNot { file ->
                val relative = repo.toPath().relativize(file.toPath()).toString().replace('\\', '/')
                EXCLUDED_ROOTS.any { relative.startsWith(it) }
            }
            .sortedBy { it.absolutePath }
            .toList()

        val reports = files.map { file ->
            val relative = repo.toPath().relativize(file.toPath()).toString().replace('\\', '/')
            report(relative, file.readText())
        }

        val totalLines = reports.sumOf { it.lines }
        val totalDeclarations = reports.sumOf { it.declarations }
        val own = KotlinSourcePartition.Kind.entries.associateWith { kind ->
            reports.sumOf { it.ownLines[kind] ?: 0 }
        }
        val fillerCodeLines = reports.sumOf { it.fillerCodeLines }
        val fillerBlankLines = reports.sumOf { it.fillerBlankLines }
        val overReports = reports.sumOf { it.overReported.size }

        println(
            "SourcePartitionGuard: partitioned ${reports.size} file(s), $totalLines line(s), " +
                "$totalDeclarations declaration(s); ${reports.sumOf { it.absorbed }} indexed " +
                "declaration(s) sit inside a signature rather than starting one"
        )
        val accounted = own.values.sum()
        fun census(label: String, count: Int) {
            val share = if (accounted == 0) 0 else count * 100 / accounted
            println("  ${label.padEnd(16)} ${count.toString().padStart(7)} line(s)  $share%")
        }
        KotlinSourcePartition.Kind.entries.forEach { kind ->
            if (kind == KotlinSourcePartition.Kind.FILLER) {
                census("filler (blank)", fillerBlankLines)
                census("filler (code)", fillerCodeLines)
            } else {
                census(kind.name.lowercase(), own[kind] ?: 0)
            }
        }
        if (accounted != totalLines) {
            System.err.println(
                "SourcePartitionGuard: the per-kind census sums to $accounted but the corpus has " +
                    "$totalLines lines — the census itself is wrong, which means a span tree double-counts"
            )
            exitProcess(1)
        }

        if (verbose) {
            println()
            println("Files with the most unattributed lines:")
            reports
                .sortedByDescending { it.fillerCodeLines }
                .take(20)
                .filter { it.fillerCodeLines > 0 }
                .forEach { println("  ${it.fillerCodeLines}\t${it.path}") }
            println()
            println("What those lines look like, in each affected file:")
            reports
                .sortedByDescending { it.fillerCodeLines }
                .filter { it.fillerCodeLines > 0 }
                .forEach { report ->
                    println("  ${report.path}")
                    report.fillerSamples.forEach { println("    $it") }
                }
            println()
            println("Most common unattributed line shapes:")
            reports
                .flatMap { it.fillerShapes }
                .groupingBy { it }
                .eachCount()
                .entries
                .sortedByDescending { it.value }
                .take(25)
                .forEach { (shape, count) -> println("  $count\t$shape") }
        }

        val violations = reports.filter { it.violations.isNotEmpty() }
        val disagreements = reports.filter { it.disagreements.isNotEmpty() }

        if (overReports > 0) {
            println()
            println(
                "$overReports declaration(s) that KotlinSourceIndex reports from inside a function " +
                    "body, in ${reports.count { it.overReported.isNotEmpty() }} file(s) — the other " +
                    "parser over-reporting, not this one losing anything:"
            )
            reports.filter { it.overReported.isNotEmpty() }.forEach { report ->
                report.overReported.forEach { println("  ${report.path}:$it") }
            }
        }

        if (violations.isNotEmpty()) {
            val total = violations.sumOf { it.violations.size }
            System.err.println()
            System.err.println("SourcePartitionGuard: $total broken invariant(s) in ${violations.size} file(s):")
            violations.take(40).forEach { report ->
                report.violations.take(10).forEach { System.err.println("  ${report.path}:$it") }
            }
        }

        if (disagreements.isNotEmpty()) {
            val total = disagreements.sumOf { it.disagreements.size }
            System.err.println()
            System.err.println(
                "SourcePartitionGuard: $total declaration(s) in ${disagreements.size} file(s) that " +
                    "KotlinSourceIndex sees and the partition cannot place inside any declaration:"
            )
            disagreements.take(40).forEach { report ->
                report.disagreements.take(10).forEach { System.err.println("  ${report.path}:$it") }
            }
        }

        if (violations.isNotEmpty() || disagreements.isNotEmpty()) {
            System.err.println(
                """
                |
                |A broken invariant means a source page rendered from this partition would drop, repeat
                |or mis-number lines. A disagreement means the source pages and the reference pages
                |would list different declarations for the same file. Fix in KotlinSourcePartition —
                |either teach the scanner the shape it cannot see, or widen FILLER to cover it, which
                |keeps the tiling true at the cost of a line the page cannot name.
                """.trimMargin()
            )
            exitProcess(1)
        }

        if (fillerCodeLines > MAX_FILLER_CODE_LINES) {
            System.err.println()
            System.err.println(
                "SourcePartitionGuard: $fillerCodeLines unattributed line(s) of code, over the ratchet " +
                    "of $MAX_FILLER_CODE_LINES. Re-run with --args=--verbose to see which shapes they " +
                    "are; either teach KotlinSourcePartition the shape, or raise MAX_FILLER_CODE_LINES " +
                    "with a reason."
            )
            exitProcess(1)
        }

        if (overReports > MAX_INDEX_OVER_REPORTS) {
            System.err.println()
            System.err.println(
                "SourcePartitionGuard: $overReports declaration(s) reported by KotlinSourceIndex from " +
                    "inside a function body, over the ratchet of $MAX_INDEX_OVER_REPORTS. Each one is a " +
                    "local that the completion panel and the reference pages will publish as though it " +
                    "were a public symbol; fix KotlinSourceIndex's brace tracking, then lower the ratchet."
            )
            exitProcess(1)
        }

        println(
            "SourcePartitionGuard ok — every file tiles, reconstructs and balances; " +
                "$fillerCodeLines unattributed line(s) of code against a ratchet of " +
                "$MAX_FILLER_CODE_LINES, $overReports index over-report(s) against a ratchet of " +
                "$MAX_INDEX_OVER_REPORTS"
        )
    }

    private fun report(path: String, text: String): FileReport {
        val partition = KotlinSourcePartition.partition(text)
        val violations = KotlinSourcePartition.verify(partition)

        val ownLines = mutableMapOf<KotlinSourcePartition.Kind, Int>()
        val fillerShapes = mutableListOf<String>()
        val fillerSamples = mutableListOf<String>()
        var fillerBlank = 0
        var fillerCode = 0
        fun account(span: KotlinSourcePartition.Span) {
            val childLines = span.children.sumOf { it.lineCount }
            val ownCount = (span.lineCount - childLines).coerceAtLeast(0)
            ownLines[span.kind] = (ownLines[span.kind] ?: 0) + ownCount
            if (span.kind == KotlinSourcePartition.Kind.FILLER) {
                for (line in span.startLine..span.endLine) {
                    val code = partition.code.getOrNull(line - 1)?.trim().orEmpty()
                    if (code.isEmpty()) fillerBlank++ else {
                        fillerCode++
                        fillerShapes += shapeOf(code)
                        if (fillerSamples.size < SAMPLES_PER_FILE) {
                            fillerSamples += "$line: ${partition.lines[line - 1].trim().take(90)}"
                        }
                    }
                }
            }
            span.children.forEach(::account)
        }
        partition.spans.forEach(::account)

        val declarationSpans = partition.allSpans()
            .filter { it.kind == KotlinSourcePartition.Kind.DECLARATION }
        val declarationLines = declarationSpans.map { it.declarationLine }.toSet()

        // A `data class Foo(val a: Int, val b: Int)` spreads its properties over the constructor
        // header, and KotlinSourceIndex reports each one as a declaration on its own line — correctly,
        // for a reference page that lists symbols. A source *page* renders the same lines as one
        // signature, so those lines are inside a declaration span rather than starting one. Accepting
        // them here is the difference between the two parsers disagreeing about a file and the two
        // parsers reading the same file for different purposes; the count is printed either way so the
        // exemption cannot quietly grow into the thing it is exempting.
        val signatureRanges = declarationSpans.map { span ->
            span.startLine..(if (span.bodyOpenLine > 0) span.bodyOpenLine else span.endLine)
        }
        val bodyRanges = declarationSpans
            .filter { it.bodyOpenLine > 0 }
            .map { it.bodyOpenLine..it.endLine }

        val indexed = try {
            KotlinSourceIndex.parseTopLevelDefinitions(text)
        } catch (t: Throwable) {
            // A crash in the other parser is that parser's problem, not evidence about this one; treat
            // the file as having no indexed declarations rather than failing this guard for it.
            emptyList()
        }
        val unmatched = indexed.filterNot { it.line in declarationLines }
        val absorbed = unmatched.count { definition -> signatureRanges.any { definition.line in it } }
        val insideBody = unmatched
            .filterNot { definition -> signatureRanges.any { definition.line in it } }
            .filter { definition -> bodyRanges.any { definition.line in it } }
        val overReported = insideBody.map {
            "${it.line}: KotlinSourceIndex sees ${it.kind} ${it.name} where the partition sees the " +
                "inside of a declaration's body"
        }
        val disagreements = unmatched
            .filterNot { definition -> signatureRanges.any { definition.line in it } }
            .filterNot { definition -> bodyRanges.any { definition.line in it } }
            .map { "${it.line}: KotlinSourceIndex sees ${it.kind} ${it.name}, the partition places that line in no declaration at all" }

        return FileReport(
            path = path,
            lines = partition.lineCount,
            ownLines = ownLines,
            declarations = declarationLines.size,
            violations = violations,
            disagreements = disagreements,
            absorbed = absorbed,
            overReported = overReported,
            fillerSamples = fillerSamples,
            fillerBlankLines = fillerBlank,
            fillerCodeLines = fillerCode,
            fillerShapes = fillerShapes,
        )
    }