meshOpCmd/polyModifierCmd.h

//-
// ==========================================================================
// Copyright 1995,2006,2008 Autodesk, Inc. All rights reserved.
//
// Use of this software is subject to the terms of the Autodesk
// license agreement provided at the time of installation or download,
// or which otherwise accompanies this software in either electronic
// or hard copy form.
// ==========================================================================
//+

#ifndef _polyModifierCmd
#define _polyModifierCmd
//
// 
// File: polyModifierCmd.h
//
// MEL Command: polyModifierCmd
//
// Author: Lonnie Li
//
// Overview:
//
//      polyModifierCmd is a generic base class designed to aid in modifying
//      polygonal meshes. All polys in Maya possess two features: construction
//      history and tweaks. Both of these have a large impact on the structure
//      of the object as well as how it can be further manipulated. However,
//      they cannot be easily implemented, which is the why we need this abstracted
//      class. polyModifierCmd will automatically handle the DG maintenance of
//      construction history and tweaks on a polygonal mesh.
//
//      To understand what effect both construction history and tweaks have on
//      a mesh, we need to understand the states of a node. There are three things
//      which will affect the state of a node regarding construction history and
//      tweaks. That is:
//
//      (1) Does construction history exist?
//      (2) Do tweaks exist?
//      (3) Is construction history turned on?
//
//      The answer to each of these questions changes how the mesh is interpreted,
//      which in turn affects how the mesh can be accessed/modified. Under each
//      circumstance, new modifications on the mesh go through a standard series
//      of events. To further illustrate how these affect the interpretation of
//      a mesh, we'll delve into each case separately looking at the problems
//      that we face in each case.
//
//      In the case where construction history exists, the existence of construction
//      history informs us that there is a single linear DG chain of nodes upstream
//      from the mesh node. That chain is the history chain. At the top of the chain
//      we have the "original" mesh and at the bottom we have the "final" mesh,
//      where "original" and "final" represent the respective state of the node with
//      regards to mesh's history. Each of these nodes are adjoined via the
//      inMesh/outMesh attributes, where in and out designate the dataflow into
//      and out of the node. Now, with that aside, attempting to modify a node
//      via mutator methods will always write the information onto the inMesh
//      attribute (except in the case of tweaks, where it writes to the cachedInMesh).
//      This presents a problem if history exists since a DG evaluation will overwrite
//      the inMesh of the mesh node, due to the connection from the outMesh of the
//      node directly upstream from the mesh. This will discard any modifications
//      made to the mesh.
//
//      So obviously modifying a mesh directly isn't possible when history exists.
//      To properly modify a mesh with history, we introduce the concept of a modifier
//      node. This polyModifierNode will encapsulate the operations on the mesh
//      and behave similarly to the other nodes in the history chain. The node will
//      then be inserted into the history chain so that on each DG evaluation, it
//      is always accounted for. The diagram below shows the before and after
//      of modifying a mesh with history.
//
//
//      Before modification:
//
//        ____                   ____
//       /    \                 /    \
//      | Hist | O --------> O | mesh | O
//       \____/  |           |  \____/  |
//            outMesh      inMesh    outMesh
//
//
//      After modification:
//
//        ____                   ________                   ____
//       /    \                 /        \                 /    \
//      | Hist | O --------> O | modifier | O --------> O | mesh | O
//       \____/  |           |  \________/  |           |  \____/  |
//            outMesh      inMesh        outMesh      inMesh    outMesh
//
//
//      (Figure 1. Nodes with History)
//
//
//      In the case of tweaks: Tweaks are stored on a hidden attribute on the
//      mesh. Tweaks are manual component modifications on a mesh (eg. repositioning
//      a vertex). During a DG evaluation, the DG takes the inMesh attribute of
//      the node and adds the tweak values onto it to get the final value. From this
//      knowledge we can see that inserting a modifier node ahead of the mesh node
//      reverses the order of operations which can be crucial to the structure of the
//      resulting mesh if the modification is a topological change. To avoid this
//      problem, we retrieve the tweaks off of the mesh, remove it from the mesh and
//      place the tweaks into a polyTweak node. The tweak node is then inserted ahead
//      of the modifier node to preserve the order of operations:
//
//
//      Before modification:
//
//                               Tweak
//        ____                   __O__
//       /    \                 /     \
//      | Hist | O --------> O | mesh  | O
//       \____/  |           |  \_____/  |
//            outMesh      inMesh     outMesh
//
//
//      After modification:
//
//                                                                      Empty Tweak
//        ____                _____                ________                __O__
//       /    \              /     \              /        \              /     \
//      | Hist | O -----> O | Tweak | O -----> O | modifier | O -----> O | mesh  | O
//       \____/  |        |  \_____/  |        |  \________/  |        |  \_____/  |
//            outMesh   inMesh     outMesh   inMesh        outMesh   inMesh     outMesh
//
//
//      (Figure 2. Node with Tweaks)
//      
//
//      The last of the questions deals with whether or not the user has construction
//      history turned on or off. This will change how the node should be modified
//      as well as what the node will look like in the DG following the operation. With
//      history turned on, the user has selected that they would like to keep a
//      history chain. So in that case, the resulting mesh would look like the above
//      diagrams following the operation. On the other hand, with history turned off,
//      the user has selected that they would not like to see a history chain. From here
//      there are two possible choices to modify the mesh:
//
//      (1) Operate on the mesh directly
//      (2) Use the DG, like in the above diagrams, then collapse the nodes down into the mesh.
//
//      The only exception to note out of this case is that if the node already possesses
//      history (as would be answered by the first question), this preference is ignored.
//      If a node has history, we continue to use history. The user is imposed with the
//      task of deleting the history on the object first if they would not like to continue
//      using history.
//
//
//      With History:
//
//        ____                   ____
//       /    \                 /    \
//      | Hist | O --------> O | mesh | O
//       \____/  |           |  \____/  |
//            outMesh      inMesh    outMesh
//
//
//      Without History:
//
//            ____
//           /    \
//        O | mesh | O     (All history compressed onto the inMesh attribute)
//        |  \____/  |
//      inMesh    outMesh
//
//
//      (Figure 3. Node with History preference)
//
//
//      This section has described the "why" part of the question regarding this command.
//      Following sections will provide a more in depth look at "how" this command
//      treats each of these situations and what it really does behind the scenes
//      to handle the above cases.
//
//
// How it works:
//
//      This command approaches the various node state cases similarly to the way
//      Maya works with construction history and tweaks in polygons. It is important
//      to note that history and tweaks are independent states having no effect on
//      each other (in terms of their state). Thus this section will describe each
//      case independently:
//
//      1) History
//
//          For history, there are 4 cases that need to be considered:
//
//          (a) History (yes)   -   RecordHistory (yes)
//          (b) History (yes)   -   RecordHistory (no)
//          (c) History (no)    -   RecordHistory (yes)
//          (d) History (no)    -   RecordHistory (no)
//
//          For (a) and (b), this command treats the node identically. Regardless of
//          whether recording history is turned on or off, if history already exists
//          on the node, we treat the node as though recording history is on. As such
//          the command performs the following steps:
//
//              (i)     Create a modifier node.
//              (ii)    Find the node directly upstream to the mesh node.
//              (iii)   Disconnect the upstream node and the mesh node.
//              (iv)    Connect the upstream node to the modifier node.
//              (v)     Connect the modifier node to the mesh node.
//              (vi)    Done!
//
//          For (c), polyModifierCmd needs to generate an input mesh to drive the
//          modifier node. To do this, the mesh node is duplicated and connected
//          like the upstream node in the previous two cases:
//
//              (i)     Create a modifier node.
//              (ii)    Duplicate the mesh node.
//              (iii)   Connect the duplicate mesh node to the modifier node
//              (iv)    Connect the modifier node to the mesh node
//              (v)     Done!
//
//          For (d), this command is a bit more complicated. There are two approaches
//          that can be done to respect the fact that no history is desired. The first
//          involves using the approach in case (c) and simply "baking" or "flattening"
//          the nodes down into the mesh node. Unfortunately, this presents some
//          serious problems with undo, as the Maya API in its current state does not
//          support construction history manipulation. Resorting to the MEL command:
//          "delete -ch" would be possible, however undoing the operation would not be
//          trivial as calling an undo from within an undo could destabilize the undo
//          queue.
//
//          The second alternative and one that is currently implemented by this class
//          is to respect the "No Construction History" preference strictly by
//          not modifying the history chain at all and simply operating directly on the
//          mesh. In order to do this and maintain generality, a hook is provided for
//          derived classes to override and place in the code used to directly modify the
//          mesh. polyModifierCmd will only call this method under the circumstances
//          of case (d). To prevent code duplication between the operations done in the
//          modifierNode and the command's directModifier implementation, the concept of
//          a factory is used. It is recommended that an instance of such a factory is
//          stored locally on the command much like it will be on the node. See
//          polyModifierNode.h and polyModifierFty.h for more details.
//
//
//      2) Tweaks
//
//          Tweaks are handled as noted above in the description section. However, how
//          they are treated is dependent on the state of history. Using the four cases
//          above:
//
//          For (a), (b) and (c), it is as described in the description section:
//
//              (i)     Create a tweak node.
//              (ii)    Extract the tweaks from the mesh node.
//              (iii)   Copy the tweaks onto the tweak node.
//              (iv)    Clear the tweaks from the mesh node.
//              (v)     Clear the tweaks from the duplicate mesh node (for case (c) only!)
//
//          For (d), we have yet another limitation. Tweaks are not handled in this case
//          because of the same circumstances which give rise to the limitation in the
//          history section. As such, topological changes may result in some odd behaviour
//          unless the workaround provided in the limitations section is used.
//
//
// How to use:
//
//      To use this command there are several things that are required based on the needs
//      of the command:
//
//      Step 1: polyModifierFty
//
//      1) Create a factory derived from polyModifierFty
//      2) Find and assign any inputs that your modifier will need onto the factory.
//      3) Override the polyModifierFty::doIt() method of the factory
//      4) Place your modifier code into the doIt() method of the factory
//
//      Step 2: polyModifierNode
//
//      1) Create a node derived from polyModifierNode
//      2) Add any additional input attributes onto the node
//      3) Associate the attributes (ie. inMesh --> affects --> outMesh)
//      4) Add an instance of your polyModifierFty to the node
//      5) Override the MPxNode::compute() method
//      6) Retrieve inputs from attributes, setup the factory and call its doIt() in compute()
//
//      Step 3: polyModifierCmd
//
//      1) Create a command derived from polyModifierCmd
//
//      ---
//
//      2) Override the polyModifierCmd::initModifierNode() method
//      3) Place your node setup code inside the initModifierNode()
//
//      ---
//
//      4) Add an instance of your polyModifierFty to the command
//      5) Cache any input parameters for the factory on the command
//      6) Override the polyModifierCmd::directModifier() method
//      7) Place your factory setup code and call its doIt() in directModifier()
//
//      ---
//
//      8) Override the MPxCommand::doIt() method
//      9) Place your setup code inside the doIt()
//      10) Place the polyModifierCmd setup code inside the doIt()
//          (ie. setMeshNode(), setModifierNodeType())
//      11) Call polyModifierCmd::doModifyPoly() inside the doIt()
//
//      ---
//
//      12) Override the MPxCommand::redoIt() method
//      13) Call polyModifierCmd::redoModifyPoly() in redoIt()
//
//      ---
//
//      14) Override the MPxCommand::undoIt() method
//      15) Call polyModifierCmd::undoModifyPoly() in undoIt()
//
//      For more details on each of these steps, please visit the associated method/class
//      headers.
//
//
// Limitations:
//
//      There is one limitation in polyModifierCmd:
//
//      (1) Duplicate mesh created under the "No History / History turned on" case not undoable
//
//      Case (1):
//
//          Under the "No History / History turned on" case, history is allowed so the DG
//          is used to perform the operation. However, every polyModifierNode requires
//          an input mesh and without any prior history, a mesh input needs to be created.
//          polyModifierCmd compensates for this by duplicating the meshNode and marking
//          it as an intermediate object.
//
//          The problem with this duplication is that the only duplicate method in the
//          Maya API resides in MFnDagNode, which does not have an associated undo/redo
//          mechanism. Attempting to manually delete the node by use of a DGmodifier or
//          the MEL delete command will break the undo/redo mechanism for the entire
//          command. As a result, this duplicate mesh is a remnant of each instance of the
//          command excluding undo/redo.
//
//          To work past this limitation, a manual delete from the MEL command line is
//          required.
//

// General Includes
//
#include <maya/MIntArray.h>
#include <maya/MFloatVectorArray.h>
#include <maya/MDagPath.h>
#include <maya/MDGModifier.h>
#include <maya/MDagModifier.h>
#include <maya/MPlug.h>

// Proxies
//
#include <maya/MPxCommand.h>


class polyModifierCmd : public MPxCommand
{
public:

                            polyModifierCmd();
                        ~polyModifierCmd() override;

// Restrict access to derived classes only
//
protected:

    // polyModifierCmd Initialization //

    // Target polyMesh to modify
    //
    void                            setMeshNode( MDagPath mesh );
    MDagPath                        getMeshNode() const;

    // Modifier Node Type
    //
    void                            setModifierNodeType( MTypeId type );
    void                            setModifierNodeName( MString name );
    MTypeId                         getModifierNodeType() const;
    MString                         getModifierNodeName() const;

    // polyModifierCmd Execution //

    // initModifierNode - Derived classes should override this method if
    //                    they wish to initialize input attributes on the 
    //                    modifierNode
    //
    virtual MStatus                 initModifierNode( MObject modifierNode );

    // directModifier - Derived classes should override this method to provide
    //                  direct modifications on the meshNode in the case where
    //                  no history exists and construction history is turned off.
    //                  (ie. no DG operations desired)
    //
    //                  This method is called only if history does not exist and
    //                  history is turned off. At this point, a handle to the
    //                  meshNode is passed in so a derived class may directly
    //                  modify the mesh.
    //
    virtual MStatus                 directModifier( MObject mesh );

    MStatus                         doModifyPoly();
    MStatus                         redoModifyPoly();
    MStatus                         undoModifyPoly();

private:

    // polyModifierCmd Internal Processing Data //

    // This structure is used to maintain the data vital to the modifyPoly method.
    // It is necessary to simplify parameter passing between the methods used inside
    // modifyPoly (specifically inside connectNodes()). The diagram below dictates
    // the naming style used:
    //
    // NOTE: modifierNode is intentionally left out of this structure since it
    //       is given protected access to derived classes.
    //
    // Before:
    //
    // (upstreamNode) *src -> dest* (meshNode)
    //
    // After:
    //
    // (upstreamNode) *src -> dest* (modifierNode) *src -> dest* (meshNode)
    //
    struct modifyPolyData
    {
        MObject meshNodeTransform;
        MObject meshNodeShape;
        MPlug   meshNodeDestPlug;
        MObject meshNodeDestAttr;

        MObject upstreamNodeTransform;
        MObject upstreamNodeShape;
        MPlug   upstreamNodeSrcPlug;
        MObject upstreamNodeSrcAttr;

        MObject modifierNodeSrcAttr;
        MObject modifierNodeDestAttr;

        MObject tweakNode;
        MObject tweakNodeSrcAttr;
        MObject tweakNodeDestAttr;
    };

    // polyModifierCmd Internal Methods //

    bool                    isCommandDataValid();
    void                    collectNodeState();

    // Modifier node methods
    //
    MStatus                 createModifierNode( MObject& modifierNode );

    // Node processing methods (need to be executed in this order)
    //
    MStatus                 processMeshNode( modifyPolyData& data );
    MStatus                 processUpstreamNode( modifyPolyData& data );
    MStatus                 processModifierNode( MObject modifierNode,
                                                 modifyPolyData& data );
    MStatus                 processTweaks( modifyPolyData& data );

    // Node connection method
    //
    MStatus                 connectNodes( MObject modifierNode );

    // Mesh caching methods - Only used in the directModifier case
    //
    MStatus                 cacheMeshData();
    MStatus                 cacheMeshTweaks();

    // Undo methods
    //
    MStatus                 undoCachedMesh();
    MStatus                 undoTweakProcessing();
    MStatus                 undoDirectModifier();

    // polyModifierCmd Utility Methods //

    MStatus                 getFloat3PlugValue( MPlug plug, MFloatVector& value );
    MStatus                 getFloat3asMObject( MFloatVector value, MObject& object );

    // polyModifierCmd Data //

    // polyMesh
    //
    bool                fDagPathInitialized;
    MDagPath            fDagPath;
    MDagPath            fDuplicateDagPath;

    // Modifier Node Type
    //
    bool                fModifierNodeTypeInitialized;
    bool                fModifierNodeNameInitialized;
    MTypeId             fModifierNodeType;
    MString             fModifierNodeName;

    // Node State Information
    //
    bool                fHasHistory;
    bool                fHasTweaks;
    bool                fHasRecordHistory;

    // Cached Tweak Data (for undo)
    //
    MIntArray           fTweakIndexArray;
    MFloatVectorArray   fTweakVectorArray;

    // Cached Mesh Data (for undo in the 'No History'/'History turned off' case)
    //
    MObject             fMeshData;

    // DG and DAG Modifier
    //
    //    - We need both DAG and DG modifiers since the MDagModifier::createNode()
    //      method is overridden and specific to DAG nodes. So to keep
    //      the operations consistent we will only use the fDagModifier
    //      when dealing with the DAG.
    //
    //    - There is a limitation between the reparentNode() and deleteNode()
    //      methods on the MDagModifier. The deleteNode() method does some
    //      preparation work before it enqueues itself in the MDagModifier list
    //      of operations, namely, it looks at it's parents and children and
    //      deletes them as well if they are the only parent/child of the node
    //      scheduled to be deleted.
    //
    //      This conflicts with our call to MDagModifier::reparentNode(),
    //      since we want to reparent the shape of a duplicated node under
    //      another node and then delete the transform of that node. Now you 
    //      can see that since the reparentNode() doesn't execute until after
    //      the MDagModifier::doIt() call, the scheduled deleteNode() call
    //      still sees the child and marks it for delete. The subsequent
    //      doIt() call reparents the shape and then deletes both it and the
    //      transform.
    //
    //      To avoid this conflict, we separate the calls individually and
    //      perform the reparenting (by calling a doIt()) before the deleteNode()
    //      method is enqueued on the modifier.
    //
    MDGModifier         fDGModifier;
    MDagModifier        fDagModifier;
};

//
// Inlines
//

// polyMesh
//
inline void polyModifierCmd::setMeshNode( MDagPath mesh )
{
    fDagPath = mesh;
    fDagPathInitialized = true;
}

inline MDagPath polyModifierCmd::getMeshNode() const
{
    return fDagPath;
}

// Modifier Node Type
//
inline void polyModifierCmd::setModifierNodeType( MTypeId type )
{
    fModifierNodeType = type;
    fModifierNodeTypeInitialized = true;
}

inline void polyModifierCmd::setModifierNodeName( MString name )
{
    fModifierNodeName = name;
    fModifierNodeNameInitialized = true;
}

inline MTypeId polyModifierCmd::getModifierNodeType() const
{
    return fModifierNodeType;
}

inline MString polyModifierCmd::getModifierNodeName() const
{
    return fModifierNodeName;
}

#endif