1

I've been experimenting with Hyperledger Composer and with the official multi org tutorial. I was successful in modifying the given demo, adding a third organisation and finally installing my own bna.

The next step was to fully understand how to deploy the Fabric network and Composer on multiple physical machines. And I went through all the available info about deploying such a process but without much luck.

Let suppose:

PC1: 1 Orderer, 1 Organisation, One cli container;

PC2: 1 Organisation;

PC3: 1 Organization;

I'm able to put the 3 machines in a swarm.

I know that I need to generate the certificates for all the machines and that they should be identical.

But from there I don't fully understand how to continue, or how to add references to the swarm network inside the compose files... 

#docker-compose-cas-template-0 - This is for PC 1
version: '2'

networks:
  example:
services:
  ca0:
    image: hyperledger/fabric-ca
    environment:
      - FABRIC_CA_HOME=/etc/hyperledger/fabric-ca-server
      - FABRIC_CA_SERVER_CA_NAME=ca-manager
      - FABRIC_CA_SERVER_TLS_ENABLED=true
      - FABRIC_CA_SERVER_TLS_CERTFILE=/etc/hyperledger/fabric-ca-server-config/ca.manager.example.com-cert.pem
      - FABRIC_CA_SERVER_TLS_KEYFILE=/etc/hyperledger/fabric-ca-server-config/CA1_PRIVATE_KEY
    ports:
      - "7054:7054"
    command: sh -c 'fabric-ca-server start --ca.certfile /etc/hyperledger/fabric-ca-server-config/ca.manager.example.com-cert.pem --ca.keyfile /etc/hyperledger/fabric-ca-server-config/CA1_PRIVATE_KEY -b admin:adminpw -d'
    volumes:
      - ./crypto-config/peerOrganizations/manager.example.com/ca/:/etc/hyperledger/fabric-ca-server-config
    container_name: ca_peerManager
    networks:
      - example



# docker-compose-base-1.yaml - This is for PC1
version: '2'

services:

  orderer.proa.com:
    container_name: orderer.proa.com
    image: hyperledger/fabric-orderer:$IMAGE_TAG
    environment:
      - ORDERER_GENERAL_LOGLEVEL=INFO
      - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
      - ORDERER_GENERAL_GENESISMETHOD=file
      - ORDERER_GENERAL_GENESISFILE=/var/hyperledger/orderer/orderer.genesis.block
      - ORDERER_GENERAL_LOCALMSPID=OrdererMSP
      - ORDERER_GENERAL_LOCALMSPDIR=/var/hyperledger/orderer/msp
      # enabled TLS
      - ORDERER_GENERAL_TLS_ENABLED=true
      - ORDERER_GENERAL_TLS_PRIVATEKEY=/var/hyperledger/orderer/tls/server.key
      - ORDERER_GENERAL_TLS_CERTIFICATE=/var/hyperledger/orderer/tls/server.crt
      - ORDERER_GENERAL_TLS_ROOTCAS=[/var/hyperledger/orderer/tls/ca.crt]
    working_dir: /opt/gopath/src/github.com/hyperledger/fabric
    command: orderer
    volumes:
    - ../channel-artifacts/genesis.block:/var/hyperledger/orderer/orderer.genesis.block
    - ../crypto-config/ordererOrganizations/proa.com/orderers/orderer.proa.com/msp:/var/hyperledger/orderer/msp
    - ../crypto-config/ordererOrganizations/proa.com/orderers/orderer.proa.com/tls/:/var/hyperledger/orderer/tls
    - orderer.proa.com:/var/hyperledger/production/orderer
    ports:
      - 7050:7050

  peer0.manager.proa.com:
    container_name: peer0.manager.proa.com
    extends:
      file: peer-base.yaml
      service: peer-base
    environment:
      - CORE_PEER_ID=peer0.manager.proa.com
      - CORE_PEER_ADDRESS=peer0.manager.proa.com:7051
      - CORE_PEER_GOSSIP_BOOTSTRAP=peer1.manager.proa.com:7051
      - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer0.manager.proa.com:7051
      - CORE_PEER_LOCALMSPID=ManagerMSP
    volumes:
        - /var/run/:/host/var/run/
        - ../crypto-config/peerOrganizations/manager.proa.com/peers/peer0.manager.proa.com/msp:/etc/hyperledger/fabric/msp
        - ../crypto-config/peerOrganizations/manager.proa.com/peers/peer0.manager.proa.com/tls:/etc/hyperledger/fabric/tls
        - peer0.manager.proa.com:/var/hyperledger/production
    ports:
      - 7051:7051
      - 7053:7053

  peer1.manager.proa.com:
    container_name: peer1.manager.proa.com
    extends:
      file: peer-base.yaml
      service: peer-base
    environment:
      - CORE_PEER_ID=peer1.manager.proa.com
      - CORE_PEER_ADDRESS=peer1.manager.proa.com:7051
      - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer1.manager.proa.com:7051
      - CORE_PEER_GOSSIP_BOOTSTRAP=peer0.manager.proa.com:7051
      - CORE_PEER_LOCALMSPID=ManagerMSP
    volumes:
        - /var/run/:/host/var/run/
        - ../crypto-config/peerOrganizations/manager.proa.com/peers/peer1.manager.proa.com/msp:/etc/hyperledger/fabric/msp
        - ../crypto-config/peerOrganizations/manager.proa.com/peers/peer1.manager.proa.com/tls:/etc/hyperledger/fabric/tls
        - peer1.manager.proa.com:/var/hyperledger/production

    ports:
      - 8051:7051
      - 8053:7053

UPDATED with docker-compose-cli.yaml - for ORG2 and PC2

# Copyright IBM Corp. All Rights Reserved.
#
# SPDX-License-Identifier: Apache-2.0
#

version: '2'

volumes:
  peer0.sponsor.example.com:
  peer1.sponsor.example.com:

networks:
  example:
    external:
      name: example

services:

  peer0.sponsor.example.com:
    container_name: peer0.sponsor.example.com
    extends:
      file:  base/docker-compose-base-2.yaml
      service: peer0.sponsor.example.com
    networks:
      - example

  peer1.sponsor.example.com:
    container_name: peer1.sponsor.example.com
    extends:
      file:  base/docker-compose-base-2.yaml
      service: peer1.sponsor.example.com
    networks:
      - example

  cli2:
    container_name: cli2
    image: hyperledger/fabric-tools:$IMAGE_TAG
    tty: true
    stdin_open: true
    environment:
      - GOPATH=/opt/gopath
      - CORE_VM_ENDPOINT=unix:///host/var/run/docker.sock
      #- CORE_LOGGING_LEVEL=DEBUG
      - CORE_LOGGING_LEVEL=INFO
      - CORE_PEER_ID=cli
      - CORE_PEER_ADDRESS=peer0.sponsor.example.com:7051
      - CORE_PEER_LOCALMSPID=SponsorMSP
      - CORE_PEER_TLS_ENABLED=true
      - CORE_PEER_TLS_CERT_FILE=/opt/gopath/src/github.com/hyperledger/fabric/peer/crypto/peerOrganizations/sponsor.example.com/peers/peer0.sponsor.example.com/tls/server.crt
      - CORE_PEER_TLS_KEY_FILE=/opt/gopath/src/github.com/hyperledger/fabric/peer/crypto/peerOrganizations/sponsor.example.com/peers/peer0.sponsor.example.com/tls/server.key
      - CORE_PEER_TLS_ROOTCERT_FILE=/opt/gopath/src/github.com/hyperledger/fabric/peer/crypto/peerOrganizations/sponsor.example.com/peers/peer0.sponsor.example.com/tls/ca.crt
      - CORE_PEER_MSPCONFIGPATH=/opt/gopath/src/github.com/hyperledger/fabric/peer/crypto/peerOrganizations/sponsor.example.com/users/Admin@sponsor.example.com/msp
    working_dir: /opt/gopath/src/github.com/hyperledger/fabric/peer
    command: /bin/bash
    volumes:
        - /var/run/:/host/var/run/
        - ./../chaincode/:/opt/gopath/src/github.com/chaincode
        - ./crypto-config:/opt/gopath/src/github.com/hyperledger/fabric/peer/crypto/
        - ./scripts:/opt/gopath/src/github.com/hyperledger/fabric/peer/scripts/
        - ./channel-artifacts:/opt/gopath/src/github.com/hyperledger/fabric/peer/channel-artifacts
    depends_on:
      - peer0.sponsor.example.com
      - peer1.sponsor.example.com

    networks:
      - example
Macaret
  • 797
  • 9
  • 33

1 Answers1

3

You're on right track. I'll list down the steps below:

  1. We create a docker swarm and connect these hosts. Since you've already created a swarm, I assume your hosts are connected.

  2. We create an overlay network from one of the hosts. In our case, it is "example" network.

    docker network create --attachable --driver overlay example

    Now, this overlay network will be available in all the hosts. You can run the following command in each of the hosts:

    docker network ls

    Here, you'll be able to see the network with the name "example" that is an overlay network.

    Also, you can inspect the network to see which all hosts (peers) are connected to this network using:

    docker network inspect example

  3. Spin up the containers. However, in this step, we need to make these containers join the existing overlay network i.e "example". So your compose files will be like: 

    version: '2'

networks:
  example:
    external:
     name: example

services:
  ca0:
    image: hyperledger/fabric-ca
    environment:
      - FABRIC_CA_HOME=/etc/hyperledger/fabric-ca-server
      - FABRIC_CA_SERVER_CA_NAME=ca-manager
      - FABRIC_CA_SERVER_TLS_ENABLED=true
      - FABRIC_CA_SERVER_TLS_CERTFILE=/etc/hyperledger/fabric-ca-server-config/ca.manager.example.com-cert.pem
      - FABRIC_CA_SERVER_TLS_KEYFILE=/etc/hyperledger/fabric-ca-server-config/CA1_PRIVATE_KEY
    ports:
      - "7054:7054"
    command: sh -c 'fabric-ca-server start --ca.certfile /etc/hyperledger/fabric-ca-server-config/ca.manager.example.com-cert.pem --ca.keyfile /etc/hyperledger/fabric-ca-server-config/CA1_PRIVATE_KEY -b admin:adminpw -d'
    volumes:
      - ./crypto-config/peerOrganizations/manager.example.com/ca/:/etc/hyperledger/fabric-ca-server-config
    container_name: ca_peerManager
    networks:
      - example

    This configuration will remain similar for all your docker containers, be it, peer, orderer, ca or cli. Also, this configuration will make sure that your container will join the existing network instead of creating a new one.

Note: Running the docker in swarm mode requires few ports to be opened up. You can find those references in this article.

Mrudav Shukla
  • 708
  • 1
  • 7
  • 25
  • 1. When I inspect the example network, I'm not seeing any host there... But if I inspect the "ingress" network I'm seeing connected all 3 hosts. 2. I'm getting: "The Docker Engine you're using is running in swarm mode. Compose does not use swarm mode to deploy services to multiple nodes in a swarm. All containers will be scheduled on the current node." – Macaret Sep 13 '18 at 14:05
  • Ok, let's go one by one. First, were you able to create the overlay network named "example" successfully? And then were you able to see that network from all the hosts? – Mrudav Shukla Sep 13 '18 at 14:20
  • Yes, I did create the network, and it's visible from all the hosts. – Macaret Sep 13 '18 at 14:25
  • Ok. So, after you spin up the containers, you'll be able to see the "Peers" when you inspect the network. For #2 of your comment, at what step are you getting that error? – Mrudav Shukla Sep 13 '18 at 14:32
  • Q1: I inspect the created overlay network (example) and I see the peers there, from all of the hosts. But on each host I'm seeing only the containers that were created there; should all the containers be visible from the all the hosts? Q2: How should I join the peers from PC-2, to the channel created by PC-1? Fetch the channel on PC-2 and join peer, or join the peers from the other hosts from PC-1 at channel creation? – Macaret Sep 14 '18 at 08:47
  • A1: So that is working fine. You'll only see containers created there. A2. You can use a "CLI" container that will contain certificates for all the peers and orderers. This "CLI" container will be on one of the hosts. From there you can create a channel and make different peers join this channel by setting the environment variables. References: https://github.com/hyperledger/fabric-samples/tree/release-1.1/first-network. Refer to byfn.sh script. – Mrudav Shukla Sep 14 '18 at 18:27
  • I created clones of docker-compose-cli.yaml for each host. One that has 1 orderer, 2 peers for org1 and cli container on PC1. For PC2 I have 2 peers for org 2 and a cli container. But I don't think this is correct way. For PC1 it works fine, but I can't join the peers on PC2 to the channel created on PC1. – Macaret Sep 17 '18 at 10:23
  • You should have a single CLI container that has crypto materials of all the peers and orderers in it. From there, you create a channel and join all the peers on this channel by updating environment variables as done over here: https://hyperledger-fabric.readthedocs.io/en/release-1.1/build_network.html#create-join-channel – Mrudav Shukla Sep 17 '18 at 10:35
  • Also, make sure all the ports used are open on both of the hosts. – Mrudav Shukla Sep 17 '18 at 10:41
  • The thing is that if I have a single docker-compose-cli file with all the peers listed on PC1, at the network up ... all the container will start, even those that should be on other hosts. And on other examples I'm seeing one cli for every the host... – Macaret Sep 17 '18 at 12:29
  • Updated with docker-compose-cli-1.yaml for organization 2 and host 2. – Macaret Sep 17 '18 at 13:22
  • So basically, on host 1 docker-compose file will contain configurations for Orderer, 2 peers, CA and CLI. Here, in the "volumes" section of the CLI conf, you'll need to pass on the crypto material for all the three orgs. And on host 2 docker-compose file will contain configurations for 2 Peers and a CA. Similarly, host 3 will contain configurations for 2 Peers and a CA. We are having a single CLI container as it will be easier to make all the peers join the channel from a single point. – Mrudav Shukla Sep 17 '18 at 14:09
  • Let us [continue this discussion in chat](https://chat.stackoverflow.com/rooms/180209/discussion-between-macaret-and-mrudav-shukla). – Macaret Sep 17 '18 at 14:21