package prosediff
import "sort"
// Tuning constants for block alignment. They are deliberately package-level
// and documented rather than hidden in the code, because they are the knobs
// that decide whether a review page reads well.
const (
// modifyThreshold is the token similarity two blocks need before they
// are called "the same block, edited" rather than a delete plus an
// insert. Below it, presenting a word diff would be noise.
modifyThreshold = 0.40
// shortBlockTokens is the size below which a block is considered too
// small to judge by similarity alone.
shortBlockTokens = 6
// shortBlockThreshold applies instead of modifyThreshold when either
// block is short: pairing "yes" with "no" on a 0.4 score helps nobody.
shortBlockThreshold = 0.60
// moveMinTokens is the smallest block that may be reported as moved.
// Below it, identical content is far more likely to be coincidence
// (a repeated "```", a "- see above") than an actual move.
moveMinTokens = 5
// pairBudget caps the similarity matrix inside one changed region. Past
// it, only pairs within pairWindow of each other are considered; a
// region with hundreds of blocks on both sides has no readable pairing
// anyway.
pairBudget = 4096
pairWindow = 8
)
// region is one changed stretch of the alignment: blocks deleted from the old
// revision and blocks inserted into the new one, adjacent in the edit script.
type region struct {
del []int // indices into old
ins []int // indices into nw
}
// align turns two block sequences into the diff's change list.
//
// Three passes, in this order:
// 1. Myers over block hashes finds the unchanged skeleton.
// 2. Blocks left over from pass 1 that reappear verbatim elsewhere are
// moves, grown outwards over their neighbours.
// 3. Similar-enough leftovers inside one changed region are modifications.
//
// Anything still unmatched is a plain insert or delete.
func align(old, nw []Block) []BlockChange {
in := newInterner()
oldIDs := make([]int, len(old))
for i, b := range old {
oldIDs[i] = in.id(b.Hash)
}
newIDs := make([]int, len(nw))
for i, b := range nw {
newIDs[i] = in.id(b.Hash)
}
script := diffInts(oldIDs, newIDs)
var (
out []BlockChange
regions []region
// slots[i] is where region i's entries go in the output.
slots []int
)
i, j := 0, 0
for k := 0; k < len(script); {
if script[k].op == OpEqual {
for n := 0; n < script[k].n; n++ {
o, w := old[i+n], nw[j+n]
out = append(out, BlockChange{Kind: ChangeEqual, Old: &o, New: &w})
}
i += script[k].n
j += script[k].n
k++
continue
}
var r region
for ; k < len(script) && script[k].op != OpEqual; k++ {
if script[k].op == OpDelete {
for n := 0; n < script[k].n; n++ {
r.del = append(r.del, i+n)
}
i += script[k].n
} else {
for n := 0; n < script[k].n; n++ {
r.ins = append(r.ins, j+n)
}
j += script[k].n
}
}
regions = append(regions, r)
slots = append(slots, len(out))
out = append(out, BlockChange{}) // placeholder, expanded below
}
moveOf, editedInMove := detectMoves(old, nw, regions)
// Build each region's entries, then splice them in at their slot.
expanded := make([][]BlockChange, len(regions))
for ri, r := range regions {
expanded[ri] = expandRegion(old, nw, r.del, r.ins, moveOf, editedInMove)
}
final := make([]BlockChange, 0, len(out)+len(regions))
next := 0
for idx := 0; idx < len(out); idx++ {
if next < len(slots) && slots[next] == idx {
final = append(final, expanded[next]...)
next++
continue
}
final = append(final, out[idx])
}
return final
}
// movePair records that old block o and new block n are the same content in a
// different place.
type movePair struct {
oldIdx int
newIdx int
}
// detectMoves matches leftover blocks across regions, seeding on exact hash
// equality and then growing each seed over its neighbours.
//
// Deliberate limitation: every move must be anchored by at least one block
// whose content is byte-identical after normalization. A block that moved and
// was edited is recognised only when it sits *between* two such anchors; a
// section that moved and was rewritten throughout falls through as a delete
// plus an insert. Matching moves by similarity alone would claim
// relationships between blocks that merely share boilerplate, and a wrong
// "moved from" costs a reviewer more than an honest add+remove.
func detectMoves(old, nw []Block, regions []region) (moves, edited map[int]movePair) {
freeOld := map[int]bool{}
freeNew := map[int]bool{}
byHash := map[string][]int{}
for _, r := range regions {
for _, oi := range r.del {
freeOld[oi] = true
if len(Tokenize(old[oi].Text)) >= moveMinTokens {
byHash[old[oi].Hash] = append(byHash[old[oi].Hash], oi)
}
}
for _, ni := range r.ins {
freeNew[ni] = true
}
}
out := map[int]movePair{}
edited = map[int]movePair{}
pair := func(oi, ni int) {
p := movePair{oldIdx: oi, newIdx: ni}
out[oldKey(oi)] = p
out[newKey(ni)] = p
delete(freeOld, oi)
delete(freeNew, ni)
}
// Seed: blocks big enough that identical content cannot be coincidence.
var seeds []movePair
for _, r := range regions {
for _, ni := range r.ins {
if len(Tokenize(nw[ni].Text)) < moveMinTokens {
continue
}
for _, oi := range byHash[nw[ni].Hash] {
if !freeOld[oi] || !freeNew[ni] {
continue
}
pair(oi, ni)
seeds = append(seeds, movePair{oi, ni})
break
}
}
}
// Grow each seed outwards while the neighbouring blocks are also
// unmatched and identical. This is what keeps a moved section's heading
// and its short trailing blocks attached to the move, instead of
// stranding them as a delete plus an insert either side of it.
//
// Growth also bridges a single edited block, but only when the block
// *past* it matches exactly — "a section was moved and one paragraph in
// it was touched" is common, while a lone similar block at the edge of a
// move is just as likely to be coincidence.
for _, s := range seeds {
for step := -1; step <= 1; step += 2 {
oi, ni := s.oldIdx+step, s.newIdx+step
for freeOld[oi] && freeNew[ni] {
if old[oi].Hash == nw[ni].Hash {
pair(oi, ni)
oi += step
ni += step
continue
}
if !bridgeable(old, nw, oi, ni, step, freeOld, freeNew, out) {
break
}
pair(oi, ni)
edited[oldKey(oi)] = movePair{oi, ni}
edited[newKey(ni)] = movePair{oi, ni}
oi += step
ni += step
}
}
}
return out, edited
}
// bridgeable reports whether old[oi] and nw[ni] are an edited version of one
// another sitting inside a run of moved blocks. The anchor past the gap may
// be either still unclaimed or already paired to its counterpart by an
// earlier seed — both mean "the move continues on the far side".
func bridgeable(old, nw []Block, oi, ni, step int, freeOld, freeNew map[int]bool, moves map[int]movePair) bool {
if old[oi].Kind != nw[ni].Kind {
return false
}
no, nn := oi+step, ni+step
if no < 0 || no >= len(old) || nn < 0 || nn >= len(nw) {
return false
}
if old[no].Hash != nw[nn].Hash {
return false
}
anchored := freeOld[no] && freeNew[nn]
if p, ok := moves[oldKey(no)]; ok && p.newIdx == nn {
anchored = true
}
if !anchored {
return false
}
return blockSimilarity(old[oi], nw[ni]) >= thresholdFor(old[oi], nw[ni])
}
// Move bookkeeping keys old and new indices into one map without colliding.
func oldKey(i int) int { return i * 2 }
func newKey(i int) int { return i*2 + 1 }
func expandRegion(old, nw []Block, del, ins []int, moveOf, editedInMove map[int]movePair) []BlockChange {
pairs := pairModified(old, nw, del, ins, moveOf)
pairedOld := map[int]int{} // old index -> new index
pairedNew := map[int]bool{}
for _, p := range pairs {
pairedOld[p.oldIdx] = p.newIdx
pairedNew[p.newIdx] = true
}
var out []BlockChange
for _, oi := range del {
o := old[oi]
if mp, ok := moveOf[oldKey(oi)]; ok {
// A block that moved and was edited is announced here and
// shown in full at its new position, where the reviewer
// reads the section it now belongs to.
n := nw[mp.newIdx]
out = append(out, BlockChange{Kind: ChangeMoveOut, Old: &o, New: &n})
continue
}
if ni, ok := pairedOld[oi]; ok {
n := nw[ni]
out = append(out, modifyChange(o, n))
continue
}
out = append(out, BlockChange{Kind: ChangeDelete, Old: &o})
}
for _, ni := range ins {
n := nw[ni]
if mp, ok := editedInMove[newKey(ni)]; ok {
c := modifyChange(old[mp.oldIdx], n)
c.Moved = true
out = append(out, c)
continue
}
if mp, ok := moveOf[newKey(ni)]; ok {
o := old[mp.oldIdx]
out = append(out, BlockChange{Kind: ChangeMoveIn, Old: &o, New: &n})
continue
}
if pairedNew[ni] {
continue // already emitted as a modification
}
out = append(out, BlockChange{Kind: ChangeInsert, New: &n})
}
return out
}
func modifyChange(o, n Block) BlockChange {
c := BlockChange{Kind: ChangeModify, Old: &o, New: &n}
if o.Kind.Prose() {
c.Words = DiffWords(o.Text, n.Text)
c.StructureOnly = !hasChange(c.Words)
} else {
c.Lines = DiffLines(o.Lines, n.Lines)
c.StructureOnly = !hasChange(c.Lines)
}
c.Similarity = blockSimilarity(o, n)
return c
}
func hasChange(spans []Span) bool {
for _, s := range spans {
if s.Op != OpEqual {
return true
}
}
return false
}
// pairModified greedily matches the most similar delete/insert pairs left in
// one changed region, best first.
func pairModified(old, nw []Block, del, ins []int, moveOf map[int]movePair) []movePair {
var cand []int
for _, oi := range del {
if _, moved := moveOf[oldKey(oi)]; !moved {
cand = append(cand, oi)
}
}
var cins []int
for _, ni := range ins {
if _, moved := moveOf[newKey(ni)]; !moved {
cins = append(cins, ni)
}
}
if len(cand) == 0 || len(cins) == 0 {
return nil
}
windowed := len(cand)*len(cins) > pairBudget
type scored struct {
p movePair
s float64
}
var all []scored
for a, oi := range cand {
for b, ni := range cins {
if windowed && abs(a-b) > pairWindow {
continue
}
o, n := old[oi], nw[ni]
if o.Kind != n.Kind {
continue
}
s := blockSimilarity(o, n)
if s < thresholdFor(o, n) {
continue
}
all = append(all, scored{movePair{oi, ni}, s})
}
}
sort.SliceStable(all, func(i, j int) bool {
if all[i].s != all[j].s {
return all[i].s > all[j].s
}
return all[i].p.oldIdx < all[j].p.oldIdx
})
usedOld := map[int]bool{}
usedNew := map[int]bool{}
var out []movePair
for _, c := range all {
if usedOld[c.p.oldIdx] || usedNew[c.p.newIdx] {
continue
}
usedOld[c.p.oldIdx] = true
usedNew[c.p.newIdx] = true
out = append(out, c.p)
}
return out
}
func thresholdFor(a, b Block) float64 {
if len(Tokenize(a.Text)) < shortBlockTokens || len(Tokenize(b.Text)) < shortBlockTokens {
return shortBlockThreshold
}
return modifyThreshold
}
// blockSimilarity is token similarity for prose and line similarity for code.
func blockSimilarity(a, b Block) float64 {
in := newInterner()
if a.Kind.Prose() {
return ratio(in.all(TokenTexts(Tokenize(a.Text))), in.all(TokenTexts(Tokenize(b.Text))))
}
return ratio(in.all(a.Lines), in.all(b.Lines))
}
func abs(i int) int {
if i < 0 {
return -i
}
return i
}