Append groups mess up everything in d3 force layout

Question

I've got a d3 force sim and if I were to add nodes as follows:

node = node.data(nodes, function(d) { return d.id;});
       node.exit().remove();
       node = node.enter().append('circle')
            .attr("class", function(d) {return d.type;})
            .attr("r", 25)
            .merge(node);

everything works fine - the circles are added at their correct places and the rendered html would look like this:

<svg width="1280" height="960">
    <g transform="translate(640,480)">
        <g stroke="#000" stroke-width="1.5">
            <line x1="197.7877989370864" y1="16.96383936157134" x2="113.39655998594978" y2="176.9054238213185"></line>
            <line x1="-99.71642802229279" y1="182.82652731678513" x2="-206.38001140055673" y2="35.62690731557146"></line>
            <line x1="-111.21899770908817" y1="-104.07607869492837" x2="9.724648489851102" y2="-238.28831674029004"></line>
            <line x1="-111.21899770908817" y1="-104.07607869492837" x2="73.66744043019104" y2="-114.11648500001087"></line>
            <line x1="197.7877989370864" y1="16.96383936157134" x2="10.328317030872993" y2="37.5171491536661"></line>
            <line x1="-99.71642802229279" y1="182.82652731678513" x2="10.328317030872993" y2="37.5171491536661"></line>
            <line x1="-111.21899770908817" y1="-104.07607869492837" x2="10.328317030872993" y2="37.5171491536661"></line>
            <line x1="197.7877989370864" y1="16.96383936157134" x2="73.66744043019104" y2="-114.11648500001087"></line>
        </g>
        <g prop="nodes" stroke="#000" stroke-width="1.5">
            <circle fill="some_image.png" class="Net" r="25" cx="197.7877989370864" cy="16.96383936157134"></circle>
            <circle fill="some_image.png" class="Net" r="25" cx="-99.71642802229279" cy="182.82652731678513"></circle>
            <circle fill="some_image.png" class="Net" r="25" cx="-111.21899770908817" cy="-104.07607869492837"></circle>
            <circle fill="some_image.png" class="Inst" r="25" cx="113.39655998594978" cy="176.9054238213185"></circle>
            <circle fill="some_image.png" class="Inst" r="25" cx="-206.38001140055673" cy="35.62690731557146"></circle>
            <circle fill="some_image.png" class="Inst" r="25" cx="9.724648489851102" cy="-238.28831674029004"></circle>
            <circle fill="some_image.png" class="Inst" r="25" cx="73.66744043019104" cy="-114.11648500001087"></circle>
            <circle fill="some_image.png" class="Internet" r="25" cx="10.328317030872993" cy="37.5171491536661"></circle>
        </g>
    </g>
</svg>

JSfiddle example

But if I wanted to add groups (my final design requires background images, labels and all sorts of additional stuff) like so:

node = node.data(nodes, function(d) { return d.id;});
       node.exit().remove();
       node.enter().append('g')
           .attr('class', 'node')
           .append('image')
           .attr('xlink:href', 'some_image.png')
           .append('text')
           .text(function(d){return d.text;})
           ... and so on...

although my code seems to get interpreted correctly (I append the groups, append the images and labels to them), the groups stay static and they remain in the middle of the sim on top of each other. Also it seems the coordinate transformation goes to the images instead to the group, which is what I think is breaking the sim:

<svg width="1280" height="960">
    <g transform="translate(640,480)">
        <g stroke="#000" stroke-width="1.5">
            <line x1="197.77682810226557" y1="16.981901068622136" x2="113.3585440445384" y2="176.90457630748227"></line>
            <line x1="-99.99450481197604" y1="182.94091641902205" x2="-206.13047480355274" y2="35.36287517221039"></line>
            <line x1="-111.19343747422879" y1="-103.71666033252438" x2="9.543859895654657" y2="-238.10758089494877"></line>
            <line x1="-111.19343747422879" y1="-103.71666033252438" x2="73.69734375869983" y2="-114.13138675745854"></line>
            <line x1="197.77682810226557" y1="16.981901068622136" x2="10.344170477990337" y2="37.84621823186521"></line>
            <line x1="-99.99450481197604" y1="182.94091641902205" x2="10.344170477990337" y2="37.84621823186521"></line>
            <line x1="-111.19343747422879" y1="-103.71666033252438" x2="10.344170477990337" y2="37.84621823186521"></line>
            <line x1="197.77682810226557" y1="16.981901068622136" x2="73.69734375869983" y2="-114.13138675745854"></line>
        </g>
    <g prop="nodes" stroke="#000" stroke-width="1.5">
        <g class="node"><image xlink:href="some_image.png" x="0" y="0" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="-7.373688780783198" y="6.754902942615239" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="1.2363864559502138" y="-14.087985964343622" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="10.538470205147267" y="13.745568221620495" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="-19.694269706308575" y="-3.4836390075862327" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="18.866941955758957" y="-12.001604111035421" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="-6.358980820385529" y="23.65509169134563" height="72" width="72" style="z-index: 3;"></image></g>
        <g class="node"><image xlink:href="some_image.png" x="-12.194453649142762" y="-23.479678451778437" height="72" width="72" style="z-index: 3;"></image></g>
        </g>
    </g>
</svg>

JSfiddle example

I'm pretty positive the use of groups messes up everything, but can't wrap my head around it how to properly use them.

Appreciate any help.

Here's the full force layout in snippet form:

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Parse tester</title>
    <script src="https://d3js.org/d3.v4.min.js" type="text/javascript"></script>
</head>
<body>
<script>
    var nodes = [
    {id:0 , label:'branch1' , name:'branch1'},
    {id:1 , label:'branch2' , name:'branch2'},
    {id:2 , label:'branch3' , name:'branch3'},
    {id:3 , label:'leaf1' , name:'leaf1'},
    {id:4 , label:'leaf2' , name:'leaf2'},
    {id:5 , label:'leaf3' , name:'leaf3'},
    {id:6 , label:'center' , name:'center'},
    {id:7 , label:'leaf23' , name:'leaf23'}
    ];
    var links = [
    {source:0 ,target:3, distance:150, weight:1},
    {source:1 ,target:4, distance:150, weight:1},
    {source:2 ,target:5, distance:150, weight:1},
    {source:7 ,target:0, distance:150, weight:1},
    {source:7 ,target:1, distance:150, weight:1},
    {source:7 ,target:2, distance:150, weight:1},
    {source:1 ,target:6, distance:150, weight:1},
    {source:2 ,target:6, distance:150, weight:1}
    ];

    //D3 stuff
    var width=640, height = 480;

    // add a SVG to the body for our viz
    var svg=d3.select('body').append('svg')
        .attr('width', width)
        .attr('height', height);

    var simulation = d3.forceSimulation(nodes)
        .force("charge", d3.forceManyBody().strength(-1000))
        .force("link", d3.forceLink(links).distance(200))
        .force("x", d3.forceX())
        .force("y", d3.forceY())
        .alphaTarget(1)
        .on("tick", ticked);

    var g = svg.append("g").attr("transform", "translate(" + width / 2 + "," + height / 2 + ")"),
        link = g.append("g").attr("stroke", "#000").attr("stroke-width", 1.5).selectAll(".link"),
        node = g.append("g").attr("stroke", "#000").attr("stroke-width", 1.5).selectAll(".node");
    
    restart();
        function restart() {

            // Apply the general update pattern to the nodes.
            node = node.data(nodes, function(d) { return d.id;});
            node.exit().remove();
            node = node.enter()
               .append('g')
               .append('image')
                  .attr('xlink:href', 'http://i.imgur.com/Rx4N3wh.png')
                  .attr('width',25)
                  .attr('height',25)
                  .attr('x', function (d) {return d.x;})
                  .attr('y', function (d) {return d.y;})
                  .merge(node);
            
            node.enter().selectAll('g').append('text')
            .attr('text-anchor', 'middle')
            .attr('dy', '.35em')
            .attr('y', -40)
            .text(function (d) {
                return d.label
            });
            
            // Apply the general update pattern to the links.
            link = link.data(links, function(d) { return d.source.id + "-" + d.target.id; });
            link.exit().remove();
            link = link.enter().append("line").merge(link);

            // Update and restart the simulation.
            simulation.nodes(nodes);
            simulation.force("link").links(links);
            simulation.alpha(1).restart();
        }
//*/


    function ticked() {
        node.attr("cx", function(d) { return d.x; })
            .attr("cy", function(d) { return d.y; });        

        link.attr("x1", function(d) { return d.source.x; })
            .attr("y1", function(d) { return d.source.y; })
            .attr("x2", function(d) { return d.target.x; })
            .attr("y2", function(d) { return d.target.y; });
    }

</script>

Answer 1

First, I got the second fiddle working, there was an extra } and a couple commas between chained methods: fiddle.

So, in the first fiddle, everything works fine from my understanding: links and nodes move about as dictated by the force layout. In fiddle two, the links continue to move, but the nodes, now gs with an image don't move whatsover.

From my understanding the key question is "Why does a g node break the force layout?" but there are also seems to be some potential questions about the tick function and nesting elements in each g node.

Force Tick Function and Update Pattern

Let's look at the tick function you use for both:

function ticked() {
    node.attr("cx", function(d) { return d.x; })
        .attr("cy", function(d) { return d.y; });        

    link.attr("x1", function(d) { return d.source.x; })
        .attr("y1", function(d) { return d.source.y; })
        .attr("x2", function(d) { return d.target.x; })
        .attr("y2", function(d) { return d.target.y; });
}

The tick function is called every tick, it updates each node based on its datum. The force simulation doesn't manipulate the visual elements whatsover, it manipulates the nodes data array. On initialization, a d3 force creates new properties for each node in the data array, such as those to represent speed and location. So d3 is updating your data, not your elements. This is why we need a tick function.

Now the above is not the typical update pattern in d3 (but it is canonical for a d3-force). A typical update pattern usually looks like:

d3.selectAll(".node")
 .data(data)
 .attr("...")
 .attr("...")...

The chain may be split with interceding exit/enter/merge selections

With objects however, which is what the nodes are in your data array, d3 does not copy the data to bind it to each element, d3 actually links each element to an item in the data array. Which means that with node.attr("cx", function(d) {, d refers to a linked/bound updated item in the data array, no need for selection.data().

I mention this because it is atypical, not well known (in my opinion), and not explained in examples or tutorials as to why a force uses (or can use) a different update pattern. Also, it may have been a source of confusion given your comment "The coords are determined by the sim and are dynamically recalculated constantly. Which is what puzzles me in the whole thing"

What is node

The selection node should be a selection of g elements, but it is in fact, a selection of image elements:

node = node.enter()      
   .append('g')      // returns a selection of `g` elements
   .append('image')  // returns a selection of `image` elements.
   ...

The elements that you are selecting aren't gs, but they are the child image elements. And merging them with other elements is likely to cause issues. Break up the chaining, keep node as a selection of your nodes, in this case your g elements. We can then append as many children as we want to each node with greater ease.

(for the sake of it, here's a fiddle with that change, but we haven't addressed why nothing is moving yet).

What breaks the Tick?

As I noted in the comments, you've changed your node from a circle to a g, and you note your nodes are initially placed, but do not update. This is because you need to change your tick function. You update nodes as so:

    node.attr("cx", function(d) { return d.x; })
        .attr("cy", function(d) { return d.y; });  

But, node is now a selection of gs, and gs aren't positioned by cx and cy attributes. Let's change that to:

node.attr("transform", function(d) { return "translate("+[d.x,d.y]+")"; })

Here's an updated fiddle with nodes moving with each udpate. But, now the positioning needs to be fixed.

Positioning

Now we update the nodes each tick, and each node's g is translated such that [0,0] is the center of that node. The images I see are 25 pixels square, so to get an image centered, we need to use negative 12.5 for the x and y positions for each image (fiddle).

I'm not using d.x or d.y to position the image like in your 2nd fiddle because by positing the g, I can position everything relative to the node much easier, and I only need to update one element for each node each tick, the g. Otherwise I'd have to update all of the labels, images, etc each tick.

The fiddle in brackets above also doesn't position the nodes initially - you can do this but a) not positioning them is easier, b) you have to be very eagle eyed to see them misplaced prior to the first tick - but some people are very eagle eyed, so there is no harm in placing them on entering (I'm not doing so for brevity here).

Why are the labels not appearing

I've left the label code as is so far as a vestigial code block (I didn't see it at first), but now we can take a closer look:

      node.enter().selectAll('g').append('text')
        .attr('text-anchor', 'middle')
        .attr('dy', '.35em')
        .attr('y', -40)
        .text(function (d) {
            return d.label
        });

node.enter() returns placeholders for each node that needs to be entered (same as when used for the g parents. But, these placeholders don't contain any gs, so node.enter().selectAll("g") will be empty, consequently text won't be appended to any element.

We want each node to have text, and each node is in the selection node, so we just use:

node.append("text")....

Here's an updated fiddle with your labels.

You can append any other children to the nodes this way, for example.

You don't need to use .data() or anything for the children because d3 will give each child the same datum as its parent.

Even if you used node.append() in the second fiddle, node represented a selection of images - and you can't append text to images - so no text would be visible.

And to keep the answer a bit more self contained, here's a snippet of the end result:

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Parse tester</title>
    <script src="https://d3js.org/d3.v4.min.js" type="text/javascript"></script>
</head>
<body>
<script>
    var nodes = [
    {id:0 , label:'branch1' , name:'branch1'},
    {id:1 , label:'branch2' , name:'branch2'},
    {id:2 , label:'branch3' , name:'branch3'},
    {id:3 , label:'leaf1' , name:'leaf1'},
    {id:4 , label:'leaf2' , name:'leaf2'},
    {id:5 , label:'leaf3' , name:'leaf3'},
    {id:6 , label:'center' , name:'center'},
    {id:7 , label:'leaf23' , name:'leaf23'}
    ];
    var links = [
    {source:0 ,target:3, distance:150, weight:1},
    {source:1 ,target:4, distance:150, weight:1},
    {source:2 ,target:5, distance:150, weight:1},
    {source:7 ,target:0, distance:150, weight:1},
    {source:7 ,target:1, distance:150, weight:1},
    {source:7 ,target:2, distance:150, weight:1},
    {source:1 ,target:6, distance:150, weight:1},
    {source:2 ,target:6, distance:150, weight:1}
    ];

    //D3 stuff
    var width=640, height = 480;

    // add a SVG to the body for our viz
    var svg=d3.select('body').append('svg')
        .attr('width', width)
        .attr('height', height);

    var simulation = d3.forceSimulation(nodes)
        .force("charge", d3.forceManyBody().strength(-1000))
        .force("link", d3.forceLink(links).distance(200))
        .force("x", d3.forceX())
        .force("y", d3.forceY())
        .alphaTarget(1)
        .on("tick", ticked);

    var g = svg.append("g").attr("transform", "translate(" + width / 2 + "," + height / 2 + ")"),
        link = g.append("g").attr("stroke", "#000").attr("stroke-width", 1.5).selectAll(".link"),
        node = g.append("g").attr("stroke", "#000").attr("stroke-width", 1.5).selectAll(".node");
    
    restart();
        function restart() {

            // Apply the general update pattern to the nodes.
            node = node.data(nodes, function(d) { return d.id;});
            node.exit().remove();
            node = node.enter()
               .append('g')
                  .attr("class","node")
                  .merge(node)
                  
                  
            node.append('image')
                  .attr('xlink:href', 'http://i.imgur.com/Rx4N3wh.png')
                  .attr('width',25)
                  .attr('height',25)
                  .attr('x', -12.5)
                  .attr('y', -12.5)
                  
            
            node.append('text')
            .attr('text-anchor', 'middle')
            .attr('dy', '.35em')
            .attr('y', -40)
            .text(function (d) {
                return d.label
            });
            
            node.append("rect")
              .attr("x", -12.5)
              .attr("y", -12.5)
              .attr("width",25)
              .attr("height",25)
              .attr("stroke-width", 4)
              .attr("stroke","steelblue")
              .attr("fill","none")
            
            // Apply the general update pattern to the links.
            link = link.data(links, function(d) { return d.source.id + "-" + d.target.id; });
            link.exit().remove();
            link = link.enter().append("line").merge(link);

            // Update and restart the simulation.
            simulation.nodes(nodes);
            simulation.force("link").links(links);
            simulation.alpha(1).restart();
        }
//*/


    function ticked() {
        node.attr("transform", function(d) { return "translate("+[d.x,d.y]+")"; })       

        link.attr("x1", function(d) { return d.source.x; })
            .attr("y1", function(d) { return d.source.y; })
            .attr("x2", function(d) { return d.target.x; })
            .attr("y2", function(d) { return d.target.y; });
    }

</script>