Environment:
java -version
openjdk version "11.0.7" 2020-04-14
OpenJDK Runtime Environment (build 11.0.7+10-post-Ubuntu-3ubuntu1)
OpenJDK 64-Bit Server VM (build 11.0.7+10-post-Ubuntu-3ubuntu1, mixed mode, sharing)
When I was using Joern (a scala program) to run a scala script (graph-for-funcs.sc) on ubuntu-20.04, in the joern interactive shell, I used a cmd like:
joern> cpg.runScript("graph-for-funcs.sc")
I ran into to the issue:
"graph-for-funcs.sc:33: object dataflowengine is not a member of package io.shiftleft"
But when I used Joern with the same script on Ubuntu-18.04, it worked. Can anyone give me some hints why this happened? I am not familiar with scala as well as java, is that because some path variables I haven't configured?
the joern program work tree:
joern/
├── graph-for-funcs.sc
└── joern-cli
├── bin
│ ├── ammonite-bridge
│ ├── ammonite-bridge.bat
│ ├── cpg-2-scpg
│ ├── cpg-2-scpg.bat
│ ├── fuzzyc-2-cpg
│ ├── fuzzyc-2-cpg.bat
│ ├── fuzzyppcli
│ ├── joern-parse
│ ├── joern-parse.bat
│ └── scalac
├── conf
│ └── log4j2.xml
├── fuzzyc2cpg.sh
├── joern
├── joern-cpg2scpg
├── joern-parse
├── lib
│ ├── au.com.bytecode.opencsv-2.4.jar
│ ├── com.carrotsearch.hppc-0.7.1.jar
│ ├── com.chuusai.shapeless_2.13-2.3.3.jar
│ ├── com.github.javaparser.javaparser-core-3.2.5.jar
│ ├── com.github.pathikrit.better-files_2.13-3.8.0.jar
│ ├── com.github.scopt.scopt_2.13-3.7.1.jar
│ ├── com.google.guava.guava-21.0.jar
│ ├── com.google.protobuf.protobuf-java-3.10.0.jar
│ ├── com.h2database.h2-mvstore-1.4.199.jar
│ ├── com.jcabi.jcabi-aspects-0.17.1.jar
│ ├── com.jcabi.jcabi-log-0.14.3.jar
│ ├── com.jcabi.jcabi-manifests-1.1.jar
│ ├── com.lihaoyi.ammonite_2.13.0-2.0.4.jar
│ ├── com.lihaoyi.ammonite-interp_2.13.0-2.0.4.jar
│ ├── com.lihaoyi.ammonite-interp-api_2.13.0-2.0.4.jar
│ ├── com.lihaoyi.ammonite-ops_2.13-2.0.4.jar
│ ├── com.lihaoyi.ammonite-repl_2.13.0-2.0.4.jar
│ ├── com.lihaoyi.ammonite-repl-api_2.13.0-2.0.4.jar
│ ├── com.lihaoyi.ammonite-runtime_2.13.0-2.0.4.jar
│ ├── com.lihaoyi.ammonite-terminal_2.13-2.0.4.jar
│ ├── com.lihaoyi.ammonite-util_2.13-2.0.4.jar
│ ├── com.lihaoyi.fansi_2.13-0.2.8.jar
│ ├── com.lihaoyi.fastparse_2.13-2.2.3.jar
│ ├── com.lihaoyi.geny_2.13-0.5.0.jar
│ ├── com.lihaoyi.os-lib_2.13-0.6.3.jar
│ ├── com.lihaoyi.pprint_2.13-0.5.8.jar
│ ├── com.lihaoyi.requests_2.13-0.5.0.jar
│ ├── com.lihaoyi.scalaparse_2.13-2.2.3.jar
│ ├── com.lihaoyi.sourcecode_2.13-0.2.0.jar
│ ├── com.lihaoyi.ujson_2.13-0.9.8.jar
│ ├── com.lihaoyi.upack_2.13-0.9.8.jar
│ ├── com.lihaoyi.upickle_2.13-0.9.8.jar
│ ├── com.lihaoyi.upickle-core_2.13-0.9.8.jar
│ ├── com.lihaoyi.upickle-implicits_2.13-0.9.8.jar
│ ├── com.massisframework.j-text-utils-0.3.4.jar
│ ├── com.michaelpollmeier.gremlin-scala_2.13-3.4.4.5.jar
│ ├── com.michaelpollmeier.macros_2.13-3.4.4.5.jar
│ ├── commons-cli.commons-cli-1.4.jar
│ ├── commons-collections.commons-collections-3.2.2.jar
│ ├── commons-configuration.commons-configuration-1.10.jar
│ ├── commons-io.commons-io-2.5.jar
│ ├── commons-lang.commons-lang-2.6.jar
│ ├── com.squareup.javapoet-1.8.0.jar
│ ├── com.thoughtworks.paranamer.paranamer-2.8.jar
│ ├── io.circe.circe-core_2.13-0.12.2.jar
│ ├── io.circe.circe-generic_2.13-0.12.2.jar
│ ├── io.circe.circe-jawn_2.13-0.12.2.jar
│ ├── io.circe.circe-numbers_2.13-0.12.2.jar
│ ├── io.circe.circe-parser_2.13-0.12.2.jar
│ ├── io.get-coursier.interface-0.0.8.jar
│ ├── io.shiftleft.codepropertygraph_2.13-0.11.287.jar
│ ├── io.shiftleft.codepropertygraph-protos_2.13-0.11.287.jar
│ ├── io.shiftleft.console_2.13-0.11.287.jar
│ ├── io.shiftleft.dataflowengineoss_2.13-0.11.287.jar
│ ├── io.shiftleft.fuzzyc2cpg_2.13-1.1.45.jar
│ ├── io.shiftleft.joern-cli-9c0397efa3eb6787d7dd1ae104bab1379a6816e8.jar
│ ├── io.shiftleft.overflowdb-tinkerpop3-0.94.jar
│ ├── io.shiftleft.overflowdb-traversal_2.13-0.94.jar
│ ├── io.shiftleft.semanticcpg_2.13-0.11.287.jar
│ ├── javax.validation.validation-api-1.1.0.Final.jar
│ ├── jline.jline-2.14.6.jar
│ ├── net.java.dev.jna.jna-4.2.2.jar
│ ├── net.objecthunter.exp4j-0.4.8.jar
│ ├── net.sf.trove4j.core-3.1.0.jar
│ ├── org.antlr.antlr4-runtime-4.7.2.jar
│ ├── org.apache.commons.commons-lang3-3.9.jar
│ ├── org.apache.logging.log4j.log4j-api-2.13.3.jar
│ ├── org.apache.logging.log4j.log4j-core-2.13.3.jar
│ ├── org.apache.logging.log4j.log4j-slf4j-impl-2.13.3.jar
│ ├── org.apache.tinkerpop.gremlin-core-3.4.4.jar
│ ├── org.apache.tinkerpop.gremlin-shaded-3.4.4.jar
│ ├── org.aspectj.aspectjrt-1.8.0.jar
│ ├── org.javassist.javassist-3.26.0-GA.jar
│ ├── org.javatuples.javatuples-1.2.jar
│ ├── org.jline.jline-reader-3.6.2.jar
│ ├── org.jline.jline-terminal-3.6.2.jar
│ ├── org.jline.jline-terminal-jna-3.6.2.jar
│ ├── org.json4s.json4s-ast_2.13-3.6.7.jar
│ ├── org.json4s.json4s-core_2.13-3.6.7.jar
│ ├── org.json4s.json4s-native_2.13-3.6.7.jar
│ ├── org.json4s.json4s-scalap_2.13-3.6.7.jar
│ ├── org.msgpack.msgpack-core-0.8.17.jar
│ ├── org.reflections.reflections-0.9.12.jar
│ ├── org.scala-lang.modules.scala-collection-compat_2.13-2.1.3.jar
│ ├── org.scala-lang.modules.scala-collection-contrib_2.13-0.2.1.jar
│ ├── org.scala-lang.modules.scala-java8-compat_2.13-0.9.0.jar
│ ├── org.scala-lang.modules.scala-parallel-collections_2.13-0.2.0.jar
│ ├── org.scala-lang.modules.scala-xml_2.13-1.2.0.jar
│ ├── org.scala-lang.scala-compiler-2.13.0.jar
│ ├── org.scala-lang.scala-library-2.13.0.jar
│ ├── org.scala-lang.scala-reflect-2.13.0.jar
│ ├── org.slf4j.jcl-over-slf4j-1.7.25.jar
│ ├── org.slf4j.slf4j-api-1.7.25.jar
│ ├── org.typelevel.cats-core_2.13-2.0.0.jar
│ ├── org.typelevel.cats-effect_2.13-2.0.0.jar
│ ├── org.typelevel.cats-kernel_2.13-2.0.0.jar
│ ├── org.typelevel.cats-macros_2.13-2.0.0.jar
│ ├── org.typelevel.jawn-parser_2.13-0.14.2.jar
│ ├── org.yaml.snakeyaml-1.15.jar
│ └── org.zeroturnaround.zt-zip-1.13.jar
├── schema-extender
│ ├── bin
│ │ ├── schema-extender
│ │ └── schema-extender.bat
│ ├── lib
│ │ ├── com.fasterxml.jackson.core.jackson-annotations-2.10.1.jar
│ │ ├── com.fasterxml.jackson.core.jackson-core-2.10.1.jar
│ │ ├── com.fasterxml.jackson.core.jackson-databind-2.10.1.jar
│ │ ├── com.fasterxml.jackson.datatype.jackson-datatype-jdk8-2.10.1.jar
│ │ ├── com.fasterxml.jackson.datatype.jackson-datatype-jsr310-2.10.1.jar
│ │ ├── com.github.pathikrit.better-files_2.12-3.8.0.jar
│ │ ├── com.github.scopt.scopt_2.12-3.7.1.jar
│ │ ├── com.lihaoyi.geny_2.12-0.4.2.jar
│ │ ├── com.lihaoyi.ujson_2.12-0.9.5.jar
│ │ ├── com.lihaoyi.upickle-core_2.12-0.9.5.jar
│ │ ├── com.typesafe.play.play-functional_2.12-2.8.1.jar
│ │ ├── com.typesafe.play.play-json_2.12-2.8.1.jar
│ │ ├── io.shiftleft.overflowdb-codegen_2.12-1.12.jar
│ │ ├── joda-time.joda-time-2.10.5.jar
│ │ ├── org.scala-lang.modules.scala-collection-compat_2.12-2.0.0.jar
│ │ ├── org.scala-lang.scala-library-2.12.11.jar
│ │ ├── org.scala-lang.scala-reflect-2.12.11.jar
│ │ ├── org.slf4j.slf4j-api-1.6.6.jar
│ │ ├── org.zeroturnaround.zt-zip-1.14.jar
│ │ └── schema-extender.schema-extender-9c0397efa3eb6787d7dd1ae104bab1379a6816e8.jar
│ └── schemas
│ ├── base.json
│ ├── closure.json
│ ├── dependency.json
│ ├── dom.json
│ ├── dynamicCalls.json
│ ├── enhancements-internal.json
│ ├── enhancements.json
│ ├── flow.json
│ ├── java-specific.json
│ ├── operators.json
│ ├── securityprofile.json
│ ├── sensitivedata.json
│ ├── source-specific.json
│ └── splitting.json
└── schema-extender.sh
graph-for-funcs.sc:
/* graph-for-funcs.scala
This script returns a Json representation of the graph resulting in combining the
AST, CGF, and PDG for each method contained in the currently loaded CPG.
Input: A valid CPG
Output: Json
Running the Script
------------------
see: README.md
The JSON generated has the following keys:
"functions": Array of all methods contained in the currently loaded CPG
|_ "function": Method name as String
|_ "id": Method id as String (String representation of the underlying Method node)
|_ "AST": see ast-for-funcs script
|_ "CFG": see cfg-for-funcs script
|_ "PDG": see pdg-for-funcs script
*/
import scala.jdk.CollectionConverters._
import io.circe.syntax._
import io.circe.generic.semiauto._
import io.circe.{Encoder, Json}
import io.shiftleft.semanticcpg.language.types.expressions.generalizations.CfgNode
import io.shiftleft.codepropertygraph.generated.EdgeTypes
import io.shiftleft.codepropertygraph.generated.NodeTypes
import io.shiftleft.codepropertygraph.generated.nodes
import io.shiftleft.dataflowengine.language._
import io.shiftleft.semanticcpg.language._
import io.shiftleft.semanticcpg.language.types.expressions.Call
import io.shiftleft.semanticcpg.language.types.structure.Local
import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn
import gremlin.scala._
import org.apache.tinkerpop.gremlin.structure.Edge
import org.apache.tinkerpop.gremlin.structure.VertexProperty
final case class GraphForFuncsFunction(function: String,
file: String,
id: String,
AST: List[nodes.AstNode],
CFG: List[nodes.AstNode],
PDG: List[nodes.AstNode])
final case class GraphForFuncsResult(functions: List[GraphForFuncsFunction])
implicit val encodeEdge: Encoder[Edge] =
(edge: Edge) =>
Json.obj(
("id", Json.fromString(edge.toString)),
("in", Json.fromString(edge.inVertex().toString)),
("out", Json.fromString(edge.outVertex().toString))
)
implicit val encodeNode: Encoder[nodes.AstNode] =
(node: nodes.AstNode) =>
Json.obj(
("id", Json.fromString(node.toString)),
("edges",
Json.fromValues((node.inE("AST", "CFG").l ++ node.outE("AST", "CFG").l).map(_.asJson))),
("properties", Json.fromValues(node.properties().asScala.toList.map { p: VertexProperty[_] =>
Json.obj(
("key", Json.fromString(p.key())),
("value", Json.fromString(p.value().toString))
)
}))
)
implicit val encodeFuncFunction: Encoder[GraphForFuncsFunction] = deriveEncoder
implicit val encodeFuncResult: Encoder[GraphForFuncsResult] = deriveEncoder
@main def main(): Json = {
GraphForFuncsResult(
cpg.method.map { method =>
val methodName = method.fullName
val methodFile = method.location.filename
val methodId = method.toString
val astChildren = method.astMinusRoot.l
val cfgChildren = new NodeSteps(
method.out(EdgeTypes.CONTAINS).filterOnEnd(_.isInstanceOf[nodes.CfgNode]).cast[nodes.CfgNode]
).l
val local = new NodeSteps(
method
.out(EdgeTypes.CONTAINS)
.hasLabel(NodeTypes.BLOCK)
.out(EdgeTypes.AST)
.hasLabel(NodeTypes.LOCAL)
.cast[nodes.Local])
val sink = local.evalType(".*").referencingIdentifiers.dedup
val source = new NodeSteps(method.out(EdgeTypes.CONTAINS).hasLabel(NodeTypes.CALL).cast[nodes.Call]).nameNot("<operator>.*").dedup
val pdgChildren = sink
.reachableByFlows(source)
.l
.flatMap { path =>
path.elements
.map {
case trackingPoint @ (_: MethodParameterIn) => trackingPoint.start.method.head
case trackingPoint => trackingPoint.cfgNode
}
}
.filter(_.toString != methodId)
GraphForFuncsFunction(methodName, methodFile, methodId, astChildren, cfgChildren, pdgChildren.distinct)
}.l
).asJson
}
