michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- michael@0: * vim: set ts=8 sts=4 et sw=4 tw=99: michael@0: * This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: #include "jit/ValueNumbering.h" michael@0: michael@0: #include "jit/IonSpewer.h" michael@0: #include "jit/MIRGenerator.h" michael@0: #include "jit/MIRGraph.h" michael@0: michael@0: using namespace js; michael@0: using namespace js::jit; michael@0: michael@0: ValueNumberer::ValueNumberer(MIRGenerator *mir, MIRGraph &graph, bool optimistic) michael@0: : mir(mir), michael@0: graph_(graph), michael@0: values(graph.alloc()), michael@0: pessimisticPass_(!optimistic), michael@0: count_(0) michael@0: { } michael@0: michael@0: TempAllocator & michael@0: ValueNumberer::alloc() const michael@0: { michael@0: return graph_.alloc(); michael@0: } michael@0: michael@0: uint32_t michael@0: ValueNumberer::lookupValue(MDefinition *ins) michael@0: { michael@0: ValueMap::AddPtr p = values.lookupForAdd(ins); michael@0: if (p) { michael@0: // make sure this is in the correct group michael@0: setClass(ins, p->key()); michael@0: return p->value(); michael@0: } michael@0: michael@0: if (!values.add(p, ins, ins->id())) michael@0: return 0; michael@0: breakClass(ins); michael@0: michael@0: return ins->id(); michael@0: } michael@0: michael@0: MDefinition * michael@0: ValueNumberer::simplify(MDefinition *def, bool useValueNumbers) michael@0: { michael@0: if (def->isEffectful()) michael@0: return def; michael@0: michael@0: MDefinition *ins = def->foldsTo(alloc(), useValueNumbers); michael@0: if (ins == def) michael@0: return def; michael@0: michael@0: // Ensure this instruction has a value number. michael@0: if (!ins->valueNumberData()) michael@0: ins->setValueNumberData(new(alloc()) ValueNumberData); michael@0: michael@0: if (!ins->block()) { michael@0: // In this case, we made a new def by constant folding, for michael@0: // example, we replaced add(#3,#4) with a new const(#7) node. michael@0: michael@0: // We will only fold a phi into one of its operands. michael@0: JS_ASSERT(!def->isPhi()); michael@0: michael@0: def->block()->insertAfter(def->toInstruction(), ins->toInstruction()); michael@0: ins->setValueNumber(lookupValue(ins)); michael@0: } michael@0: michael@0: JS_ASSERT(ins->id() != 0); michael@0: michael@0: def->replaceAllUsesWith(ins); michael@0: michael@0: IonSpew(IonSpew_GVN, "Folding %d to be %d", def->id(), ins->id()); michael@0: return ins; michael@0: } michael@0: michael@0: MControlInstruction * michael@0: ValueNumberer::simplifyControlInstruction(MControlInstruction *def) michael@0: { michael@0: if (def->isEffectful()) michael@0: return def; michael@0: michael@0: MDefinition *repl = def->foldsTo(alloc(), false); michael@0: if (repl == def) michael@0: return def; michael@0: michael@0: // Ensure this instruction has a value number. michael@0: if (!repl->valueNumberData()) michael@0: repl->setValueNumberData(new(alloc()) ValueNumberData); michael@0: michael@0: MBasicBlock *block = def->block(); michael@0: michael@0: // MControlInstructions should not have consumers. michael@0: JS_ASSERT(repl->isControlInstruction()); michael@0: JS_ASSERT(!def->hasUses()); michael@0: michael@0: if (def->isInWorklist()) michael@0: repl->setInWorklist(); michael@0: michael@0: block->discardLastIns(); michael@0: block->end((MControlInstruction *)repl); michael@0: return (MControlInstruction *)repl; michael@0: } michael@0: michael@0: void michael@0: ValueNumberer::markDefinition(MDefinition *def) michael@0: { michael@0: if (isMarked(def)) michael@0: return; michael@0: michael@0: IonSpew(IonSpew_GVN, "marked %d", def->id()); michael@0: def->setInWorklist(); michael@0: count_++; michael@0: } michael@0: michael@0: void michael@0: ValueNumberer::unmarkDefinition(MDefinition *def) michael@0: { michael@0: if (pessimisticPass_) michael@0: return; michael@0: michael@0: JS_ASSERT(count_ > 0); michael@0: IonSpew(IonSpew_GVN, "unmarked %d", def->id()); michael@0: def->setNotInWorklist(); michael@0: count_--; michael@0: } michael@0: michael@0: void michael@0: ValueNumberer::markBlock(MBasicBlock *block) michael@0: { michael@0: for (MDefinitionIterator iter(block); iter; iter++) michael@0: markDefinition(*iter); michael@0: markDefinition(block->lastIns()); michael@0: } michael@0: michael@0: void michael@0: ValueNumberer::markConsumers(MDefinition *def) michael@0: { michael@0: if (pessimisticPass_) michael@0: return; michael@0: michael@0: JS_ASSERT(!def->isInWorklist()); michael@0: JS_ASSERT(!def->isControlInstruction()); michael@0: for (MUseDefIterator use(def); use; use++) michael@0: markDefinition(use.def()); michael@0: } michael@0: michael@0: bool michael@0: ValueNumberer::computeValueNumbers() michael@0: { michael@0: // At the end of this function, we will have the value numbering stored in michael@0: // each instruction. michael@0: // michael@0: // We also need an "optimistic" value number, for temporary use, which is michael@0: // stored in a hashtable. michael@0: // michael@0: // For the instruction x := y op z, we map (op, VN[y], VN[z]) to a value michael@0: // number, say v. If it is not in the map, we use the instruction id. michael@0: // michael@0: // If the instruction in question's value number is not already michael@0: // v, we break the congruence and set it to v. We repeat until saturation. michael@0: // This will take at worst O(d) time, where d is the loop connectedness michael@0: // of the SSA def/use graph. michael@0: // michael@0: // The algorithm is the simple RPO-based algorithm from michael@0: // "SCC-Based Value Numbering" by Cooper and Simpson. michael@0: // michael@0: // If we are performing a pessimistic pass, then we assume that every michael@0: // definition is in its own congruence class, since we know nothing about michael@0: // values that enter Phi nodes through back edges. We then make one pass michael@0: // through the graph, ignoring back edges. This yields less congruences on michael@0: // any graph with back-edges, but is much faster to perform. michael@0: michael@0: IonSpew(IonSpew_GVN, "Numbering instructions"); michael@0: michael@0: if (!values.init()) michael@0: return false; michael@0: // Stick a VN object onto every mdefinition michael@0: for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { michael@0: if (mir->shouldCancel("Value Numbering (preparation loop")) michael@0: return false; michael@0: for (MDefinitionIterator iter(*block); iter; iter++) michael@0: iter->setValueNumberData(new(alloc()) ValueNumberData); michael@0: MControlInstruction *jump = block->lastIns(); michael@0: jump->setValueNumberData(new(alloc()) ValueNumberData); michael@0: } michael@0: michael@0: // Assign unique value numbers if pessimistic. michael@0: // It might be productive to do this in the MDefinition constructor or michael@0: // possibly in a previous pass, if it seems reasonable. michael@0: if (pessimisticPass_) { michael@0: for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { michael@0: for (MDefinitionIterator iter(*block); iter; iter++) michael@0: iter->setValueNumber(iter->id()); michael@0: } michael@0: } else { michael@0: // For each root block, add all of its instructions to the worklist. michael@0: markBlock(*(graph_.begin())); michael@0: if (graph_.osrBlock()) michael@0: markBlock(graph_.osrBlock()); michael@0: } michael@0: michael@0: while (count_ > 0) { michael@0: #ifdef DEBUG michael@0: if (!pessimisticPass_) { michael@0: size_t debugCount = 0; michael@0: IonSpew(IonSpew_GVN, "The following instructions require processing:"); michael@0: for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { michael@0: for (MDefinitionIterator iter(*block); iter; iter++) { michael@0: if (iter->isInWorklist()) { michael@0: IonSpew(IonSpew_GVN, "\t%d", iter->id()); michael@0: debugCount++; michael@0: } michael@0: } michael@0: if (block->lastIns()->isInWorklist()) { michael@0: IonSpew(IonSpew_GVN, "\t%d", block->lastIns()->id()); michael@0: debugCount++; michael@0: } michael@0: } michael@0: if (!debugCount) michael@0: IonSpew(IonSpew_GVN, "\tNone"); michael@0: JS_ASSERT(debugCount == count_); michael@0: } michael@0: #endif michael@0: for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { michael@0: if (mir->shouldCancel("Value Numbering (main loop)")) michael@0: return false; michael@0: for (MDefinitionIterator iter(*block); iter; ) { michael@0: michael@0: if (!isMarked(*iter)) { michael@0: iter++; michael@0: continue; michael@0: } michael@0: michael@0: JS_ASSERT_IF(!pessimisticPass_, count_ > 0); michael@0: unmarkDefinition(*iter); michael@0: michael@0: MDefinition *ins = simplify(*iter, false); michael@0: michael@0: if (ins != *iter) { michael@0: iter = block->discardDefAt(iter); michael@0: continue; michael@0: } michael@0: michael@0: // Don't bother storing this instruction in the HashMap if michael@0: // (a) eliminateRedundancies will never eliminate it (because michael@0: // it's non-movable or effectful) and (b) no other instruction's michael@0: // value number depends on it. michael@0: if (!ins->hasDefUses() && (!ins->isMovable() || ins->isEffectful())) { michael@0: iter++; michael@0: continue; michael@0: } michael@0: michael@0: uint32_t value = lookupValue(ins); michael@0: michael@0: if (!value) michael@0: return false; // Hashtable insertion failed michael@0: michael@0: if (ins->valueNumber() != value) { michael@0: IonSpew(IonSpew_GVN, michael@0: "Broke congruence for instruction %d (%p) with VN %d (now using %d)", michael@0: ins->id(), (void *) ins, ins->valueNumber(), value); michael@0: ins->setValueNumber(value); michael@0: markConsumers(ins); michael@0: } michael@0: michael@0: iter++; michael@0: } michael@0: // Process control flow instruction: michael@0: MControlInstruction *jump = block->lastIns(); michael@0: jump = simplifyControlInstruction(jump); michael@0: michael@0: // If we are pessimistic, then this will never get set. michael@0: if (!jump->isInWorklist()) michael@0: continue; michael@0: unmarkDefinition(jump); michael@0: if (jump->valueNumber() == 0) { michael@0: jump->setValueNumber(jump->id()); michael@0: for (size_t i = 0; i < jump->numSuccessors(); i++) michael@0: markBlock(jump->getSuccessor(i)); michael@0: } michael@0: michael@0: } michael@0: michael@0: // If we are doing a pessimistic pass, we only go once through the michael@0: // instruction list. michael@0: if (pessimisticPass_) michael@0: break; michael@0: } michael@0: #ifdef DEBUG michael@0: for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { michael@0: for (MDefinitionIterator iter(*block); iter; iter++) { michael@0: JS_ASSERT(!iter->isInWorklist()); michael@0: JS_ASSERT_IF(iter->valueNumber() == 0, michael@0: !iter->hasDefUses() && (!iter->isMovable() || iter->isEffectful())); michael@0: } michael@0: } michael@0: #endif michael@0: return true; michael@0: } michael@0: michael@0: MDefinition * michael@0: ValueNumberer::findDominatingDef(InstructionMap &defs, MDefinition *ins, size_t index) michael@0: { michael@0: JS_ASSERT(ins->valueNumber() != 0); michael@0: InstructionMap::Ptr p = defs.lookup(ins->valueNumber()); michael@0: MDefinition *dom; michael@0: if (!p || index > p->value().validUntil) { michael@0: DominatingValue value; michael@0: value.def = ins; michael@0: // Since we are traversing the dominator tree in pre-order, when we michael@0: // are looking at the |index|-th block, the next numDominated() blocks michael@0: // we traverse are precisely the set of blocks that are dominated. michael@0: // michael@0: // So, this value is visible in all blocks if: michael@0: // index <= index + ins->block->numDominated() michael@0: // and becomes invalid after that. michael@0: value.validUntil = index + ins->block()->numDominated(); michael@0: michael@0: if(!defs.put(ins->valueNumber(), value)) michael@0: return nullptr; michael@0: michael@0: dom = ins; michael@0: } else { michael@0: dom = p->value().def; michael@0: } michael@0: michael@0: return dom; michael@0: } michael@0: michael@0: bool michael@0: ValueNumberer::eliminateRedundancies() michael@0: { michael@0: // A definition is 'redundant' iff it is dominated by another definition michael@0: // with the same value number. michael@0: // michael@0: // So, we traverse the dominator tree in pre-order, maintaining a hashmap michael@0: // from value numbers to instructions. michael@0: // michael@0: // For each definition d with value number v, we look up v in the hashmap. michael@0: // michael@0: // If there is a definition d' in the hashmap, and the current traversal michael@0: // index is within that instruction's dominated range, then we eliminate d, michael@0: // replacing all uses of d with uses of d'. michael@0: // michael@0: // If there is no valid definition in the hashtable (the current definition michael@0: // is not in dominated scope), then we insert the current instruction, michael@0: // since it is the most dominant instruction with the given value number. michael@0: michael@0: InstructionMap defs(alloc()); michael@0: michael@0: if (!defs.init()) michael@0: return false; michael@0: michael@0: IonSpew(IonSpew_GVN, "Eliminating redundant instructions"); michael@0: michael@0: // Stack for pre-order CFG traversal. michael@0: Vector worklist(alloc()); michael@0: michael@0: // The index of the current block in the CFG traversal. michael@0: size_t index = 0; michael@0: michael@0: // Add all self-dominating blocks to the worklist. michael@0: // This includes all roots. Order does not matter. michael@0: for (MBasicBlockIterator i(graph_.begin()); i != graph_.end(); i++) { michael@0: MBasicBlock *block = *i; michael@0: if (block->immediateDominator() == block) { michael@0: if (!worklist.append(block)) michael@0: return false; michael@0: } michael@0: } michael@0: michael@0: // Starting from each self-dominating block, traverse the CFG in pre-order. michael@0: while (!worklist.empty()) { michael@0: if (mir->shouldCancel("Value Numbering (eliminate loop)")) michael@0: return false; michael@0: MBasicBlock *block = worklist.popCopy(); michael@0: michael@0: IonSpew(IonSpew_GVN, "Looking at block %d", block->id()); michael@0: michael@0: // Add all immediate dominators to the front of the worklist. michael@0: if (!worklist.append(block->immediatelyDominatedBlocksBegin(), michael@0: block->immediatelyDominatedBlocksEnd())) { michael@0: return false; michael@0: } michael@0: michael@0: // For each instruction, attempt to look up a dominating definition. michael@0: for (MDefinitionIterator iter(block); iter; ) { michael@0: MDefinition *ins = simplify(*iter, true); michael@0: michael@0: // Instruction was replaced, and all uses have already been fixed. michael@0: if (ins != *iter) { michael@0: iter = block->discardDefAt(iter); michael@0: continue; michael@0: } michael@0: michael@0: // Instruction has side-effects and cannot be folded. michael@0: if (!ins->isMovable() || ins->isEffectful()) { michael@0: iter++; michael@0: continue; michael@0: } michael@0: michael@0: MDefinition *dom = findDominatingDef(defs, ins, index); michael@0: if (!dom) michael@0: return false; // Insertion failed. michael@0: michael@0: if (dom == ins || !dom->updateForReplacement(ins)) { michael@0: iter++; michael@0: continue; michael@0: } michael@0: michael@0: IonSpew(IonSpew_GVN, "instruction %d is dominated by instruction %d (from block %d)", michael@0: ins->id(), dom->id(), dom->block()->id()); michael@0: michael@0: ins->replaceAllUsesWith(dom); michael@0: michael@0: JS_ASSERT(!ins->hasUses()); michael@0: JS_ASSERT(ins->block() == block); michael@0: JS_ASSERT(!ins->isEffectful()); michael@0: JS_ASSERT(ins->isMovable()); michael@0: michael@0: iter = ins->block()->discardDefAt(iter); michael@0: } michael@0: index++; michael@0: } michael@0: michael@0: JS_ASSERT(index == graph_.numBlocks()); michael@0: return true; michael@0: } michael@0: michael@0: // Exported method, called by the compiler. michael@0: bool michael@0: ValueNumberer::analyze() michael@0: { michael@0: return computeValueNumbers() && eliminateRedundancies(); michael@0: } michael@0: michael@0: // Called by the compiler if we need to re-run GVN. michael@0: bool michael@0: ValueNumberer::clear() michael@0: { michael@0: IonSpew(IonSpew_GVN, "Clearing value numbers"); michael@0: michael@0: // Clear the VN of every MDefinition michael@0: for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { michael@0: if (mir->shouldCancel("Value Numbering (clearing)")) michael@0: return false; michael@0: for (MDefinitionIterator iter(*block); iter; iter++) michael@0: iter->clearValueNumberData(); michael@0: block->lastIns()->clearValueNumberData(); michael@0: } michael@0: michael@0: return true; michael@0: } michael@0: michael@0: uint32_t michael@0: MDefinition::valueNumber() const michael@0: { michael@0: JS_ASSERT(block_); michael@0: if (valueNumber_ == nullptr) michael@0: return 0; michael@0: return valueNumber_->valueNumber(); michael@0: } michael@0: void michael@0: MDefinition::setValueNumber(uint32_t vn) michael@0: { michael@0: valueNumber_->setValueNumber(vn); michael@0: } michael@0: // Set the class of this to the given representative value. michael@0: void michael@0: ValueNumberer::setClass(MDefinition *def, MDefinition *rep) michael@0: { michael@0: def->valueNumberData()->setClass(def, rep); michael@0: } michael@0: michael@0: MDefinition * michael@0: ValueNumberer::findSplit(MDefinition *def) michael@0: { michael@0: for (MDefinition *vncheck = def->valueNumberData()->classNext; michael@0: vncheck != nullptr; michael@0: vncheck = vncheck->valueNumberData()->classNext) { michael@0: if (!def->congruentTo(vncheck)) { michael@0: IonSpew(IonSpew_GVN, "Proceeding with split because %d is not congruent to %d", michael@0: def->id(), vncheck->id()); michael@0: return vncheck; michael@0: } michael@0: } michael@0: return nullptr; michael@0: } michael@0: michael@0: void michael@0: ValueNumberer::breakClass(MDefinition *def) michael@0: { michael@0: if (def->valueNumber() == def->id()) { michael@0: IonSpew(IonSpew_GVN, "Breaking congruence with itself: %d", def->id()); michael@0: ValueNumberData *defdata = def->valueNumberData(); michael@0: JS_ASSERT(defdata->classPrev == nullptr); michael@0: // If the def was the only member of the class, then there is nothing to do. michael@0: if (defdata->classNext == nullptr) michael@0: return; michael@0: // If upon closer inspection, we are still equivalent to this class michael@0: // then there isn't anything for us to do. michael@0: MDefinition *newRep = findSplit(def); michael@0: if (!newRep) michael@0: return; michael@0: markConsumers(def); michael@0: ValueNumberData *newdata = newRep->valueNumberData(); michael@0: michael@0: // Right now, |defdata| is at the front of the list, and |newdata| is michael@0: // somewhere in the middle. michael@0: // michael@0: // We want to move |defdata| and everything up to but excluding michael@0: // |newdata| to a new list, with |defdata| still as the canonical michael@0: // element. michael@0: // michael@0: // We then want to take |newdata| and everything after, and michael@0: // mark them for processing (since |newdata| is now a new canonical michael@0: // element). michael@0: // michael@0: MDefinition *lastOld = newdata->classPrev; michael@0: michael@0: JS_ASSERT(lastOld); // newRep is NOT the first element of the list. michael@0: JS_ASSERT(lastOld->valueNumberData()->classNext == newRep); michael@0: michael@0: //lastOld is now the last element of the old list (congruent to michael@0: //|def|) michael@0: lastOld->valueNumberData()->classNext = nullptr; michael@0: michael@0: #ifdef DEBUG michael@0: for (MDefinition *tmp = def; tmp != nullptr; tmp = tmp->valueNumberData()->classNext) { michael@0: JS_ASSERT(tmp->valueNumber() == def->valueNumber()); michael@0: JS_ASSERT(tmp->congruentTo(def)); michael@0: JS_ASSERT(tmp != newRep); michael@0: } michael@0: #endif michael@0: //|newRep| is now the first element of a new list, therefore it is the michael@0: //new canonical element. Mark the remaining elements in the list michael@0: //(including |newRep|) michael@0: newdata->classPrev = nullptr; michael@0: IonSpew(IonSpew_GVN, "Choosing a new representative: %d", newRep->id()); michael@0: michael@0: // make the VN of every member in the class the VN of the new representative number. michael@0: for (MDefinition *tmp = newRep; tmp != nullptr; tmp = tmp->valueNumberData()->classNext) { michael@0: // if this instruction is already scheduled to be processed, don't do anything. michael@0: if (tmp->isInWorklist()) { michael@0: IonSpew(IonSpew_GVN, "Defer to a new congruence class: %d", tmp->id()); michael@0: continue; michael@0: } michael@0: IonSpew(IonSpew_GVN, "Moving to a new congruence class: %d", tmp->id()); michael@0: tmp->setValueNumber(newRep->id()); michael@0: markConsumers(tmp); michael@0: markDefinition(tmp); michael@0: } michael@0: michael@0: // Insert the new representative => number mapping into the table michael@0: // Logically, there should not be anything in the table currently, but michael@0: // old values are never removed, so there's a good chance something will michael@0: // already be there. michael@0: values.put(newRep, newRep->id()); michael@0: } else { michael@0: // The element that is breaking from the list isn't the representative element michael@0: // just strip it from the list michael@0: ValueNumberData *defdata = def->valueNumberData(); michael@0: if (defdata->classPrev) michael@0: defdata->classPrev->valueNumberData()->classNext = defdata->classNext; michael@0: if (defdata->classNext) michael@0: defdata->classNext->valueNumberData()->classPrev = defdata->classPrev; michael@0: michael@0: // Make sure there is no nastinees accidentally linking elements into the old list later. michael@0: defdata->classPrev = nullptr; michael@0: defdata->classNext = nullptr; michael@0: } michael@0: }