Edge-LB

Follow the documentation available here to deploy Edge-LB to your cluster.

Create DC/OS Data Science Engine config dse.json specifying hostname:

{
  "service": {
    "name" : "dse"
  },
  "networking": {
    "ingress": {
      "enabled": true,
      "hostname": "external.host.com"
    }
  }
}

Deploy DC/OS Data Science Engine:

dcos package install data-science-engine --options=dse.json

Create Edge-LB pool configuration file dse-pool.json.

For HTTPS, ssl certificates should be specified:

  "secrets": [
    {
      "secret": "dse-sslcert",
      "file": "dse-sslcert"
    }
  ],

For port discovery point backend to DC/OS Data Science Engine notebook task:

 ...
 "mesos" : {
    "frameworkName": "dse",
    "taskName": "data-science-engine-0-notebook"
 },
 ...

SparkUI should have additional backend rules to handle Jobs and Executors pages:

sparkuiJobs:


    "name": "sparkuiJobs",
    "protocol": "HTTP",
    "rewriteHttp": {
      "path": {
        "fromPath": "/sparkui/jobs",
        "toPath": "/jobs/"
      },
      "request" : {
        "rewritePath": false
      }

sparkuiApplications:

    "name": "sparkuiApplications",
    "protocol": "HTTP",
    "rewriteHttp": {
      "path": {
        "fromPath": "/api/v1/",
        "toPath": "/sparkui/api/v1/"
      },
      "request" : {
        "rewritePath": false
      }

sparkuiExecutorspage:

    "name": "sparkuiExecutorspage",
    "protocol": "HTTP",
    "rewriteHttp": {
      "path": {
        "fromPath": "/static/executorspage-template.html",
        "toPath": "/sparkui/static/executorspage-template.html"
      },
      "request" : {
        "rewritePath": false
      }

Example Edge-LB pool configuration dse-pool.json for DC/OS Data Science Engine named dse and external hostname external.host.com:

{
  "apiVersion": "V2",
  "name": "dse-pool",
  "count": 1,
  "secrets": [
    {
      "secret": "dse-sslcert",
      "file": "dse-sslcert"
    }
  ],
  "haproxy": {
    "frontends": [
      {
        "bindPort": 80,
        "protocol": "HTTP",
        "redirectToHttps": {
          "items": {
            "host": "external.host.com"
          }
        }
      },
      {
        "bindPort": 443,
        "protocol": "HTTPS",
        "certificates": [
          "$SECRETS/dse-sslcert"
        ],
        "linkBackend": {
          "defaultBackend" : "notebook",
          "map": [
            {
              "hostEq": "external.host.com",
              "backend": "tensorboard",
              "pathBeg": "/tensorboard"
            },
            {
              "hostEq": "external.host.com",
              "backend": "sparkhistory",
              "pathBeg": "/sparkhistory"
            },
            {
              "hostEq": "external.host.com",
              "backend": "sparkui",
              "pathBeg": "/sparkui"
            },
            {
              "hostEq": "external.host.com",
              "backend": "sparkuiJobs",
              "pathBeg": "/jobs"
            },
            {
              "hostEq": "external.host.com",
              "backend": "sparkuiApplications",
              "pathBeg": "/api/v1/"
            },
            {
              "hostEq": "external.host.com",
              "backend": "sparkuiExecutorspage",
              "pathBeg": "/static/executorspage-template.html"
            }
          ]
        }
      }
    ],
    "backends": [
      {
        "name": "notebook",
        "protocol": "HTTP",
        "services": [
          {
            "mesos" : {
              "frameworkName": "dse",
              "taskName": "data-science-engine-0-notebook"
            },
            "endpoint": {
              "type": "AUTO_IP",
              "portName": "notebook"
            }
          }
        ]
      },
       {
         "name": "tensorboard",
         "protocol": "HTTP",
         "services": [
           {
             "mesos" : {
               "frameworkName": "dse",
               "taskName": "data-science-engine-0-notebook"
             },
             "endpoint": {
               "type": "AUTO_IP",
               "portName": "tensorboard"
             }
           }
         ]
       },
      {
        "name": "sparkhistory",
        "protocol": "HTTP",
        "rewriteHttp": {
          "path": {
            "fromPath": "/sparkhistory/",
            "toPath": "/"
          }
        },
        "services": [
          {
            "mesos" : {
              "frameworkName": "dse",
              "taskName": "data-science-engine-0-notebook"
            },
            "endpoint": {
              "type": "AUTO_IP",
              "portName": "sparkhistory"
            }
          }
        ]
      },
      {
        "name": "sparkui",
        "protocol": "HTTP",
        "rewriteHttp": {
          "path": {
            "fromPath": "/sparkui",
            "toPath": "/"
          },
          "response" : {
            "rewriteLocation": false
          }
        },
        "services": [
          {
            "mesos" : {
              "frameworkName": "dse",
              "taskName": "data-science-engine-0-notebook"
            },
            "endpoint": {
              "type": "AUTO_IP",
              "portName": "sparkui"
            }
          }
        ]
      },
      {
        "name": "sparkuiJobs",
        "protocol": "HTTP",
        "rewriteHttp": {
          "path": {
            "fromPath": "/sparkui/jobs",
            "toPath": "/jobs/"
          },
          "request" : {
            "rewritePath": false
          }
        },
        "services": [
          {
            "mesos" : {
              "frameworkName": "dse",
              "taskName": "data-science-engine-0-notebook"
            },
            "endpoint": {
              "type": "AUTO_IP",
              "portName": "sparkui"
            }
          }
        ]
      },
      {
        "name": "sparkuiApplications",
        "protocol": "HTTP",
        "rewriteHttp": {
          "path": {
            "fromPath": "/api/v1/",
            "toPath": "/sparkui/api/v1/"
          },
          "request" : {
            "rewritePath": false
          }
        },
        "services": [
          {
            "mesos" : {
              "frameworkName": "dse",
              "taskName": "data-science-engine-0-notebook"
            },
            "endpoint": {
              "type": "AUTO_IP",
              "portName": "sparkui"
            }
          }
        ]
      },
      {
        "name": "sparkuiExecutorspage",
        "protocol": "HTTP",
        "rewriteHttp": {
          "path": {
            "fromPath": "/static/executorspage-template.html",
            "toPath": "/sparkui/static/executorspage-template.html"
          },
          "request" : {
            "rewritePath": false
          }
        },
        "services": [
          {
            "mesos" : {
              "frameworkName": "dse",
              "taskName": "data-science-engine-0-notebook"
            },
            "endpoint": {
              "type": "AUTO_IP",
              "portName": "sparkui"
            }
          }
        ]
      }
    ]
  }
}

Add permissions for Edge-LB pool dse-pool:

dcos security org users grant edge-lb-principal
dcos:adminrouter:service:dcos-edgelb/pools/dse-pool full

Deploy Edge-LB Pool:
```
dcos edgelb create dse-pool.json
```
Confirm that our Edge-LB pool can see the backend and the backend is healthy by checking
```
<agent-ip>:9090/haproxy?stats
```
For dse-pool.json example access URLs will be:
- Jupyter Notebook UI - https://external.host.com/service/dse/
- TensorBoard - https://external.host.com/tensorboard
- SparkHistory - https://external.host.com/sparkhistory
- SparkUI - https://external.host.com/sparkui/

Accessing the DC/OS Data Science Engine with Edge-LB configuration