We use scala programing language and use spark as a data processing. we have 3 different projects
Project A // where we have defined this annotation
package Common {
case class TimeStamp(format: DateFormat) extends StaticAnnotation
}
Project B // where we have this case class which is using above annotation
case class CommonSchema(
@Common.TimeStamp(DateFormat.DateTime)
__datetime: Timestamp
)
Project C // where we use above case class as type
val events = events.map(c => CommonSchema(
c.__datetime,
))
events.as[CommonSchema]
We are hitting the below exception in project C
User class threw exception: java.lang.RuntimeException: error reading Scala signature of commonschema: assertion failed: unsafe symbol TimeStamp (child of <none>) in runtime reflection universe
at scala.reflect.internal.pickling.UnPickler.unpickle(UnPickler.scala:46)
at scala.reflect.runtime.JavaMirrors$JavaMirror.unpickleClass(JavaMirrors.scala:619)
at scala.reflect.runtime.SymbolLoaders$TopClassCompleter$$anonfun$complete$1.apply$mcV$sp(SymbolLoaders.scala:28)
at scala.reflect.runtime.SymbolLoaders$TopClassCompleter$$anonfun$complete$1.apply(SymbolLoaders.scala:25)
at scala.reflect.runtime.SymbolLoaders$TopClassCompleter$$anonfun$complete$1.apply(SymbolLoaders.scala:25)
at scala.reflect.internal.SymbolTable.slowButSafeEnteringPhaseNotLaterThan(SymbolTable.scala:263)
at scala.reflect.runtime.SymbolLoaders$TopClassCompleter.complete(SymbolLoaders.scala:25)
at scala.reflect.runtime.SymbolLoaders$TopClassCompleter.load(SymbolLoaders.scala:33)
at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anonfun$typeParams$1.apply(SynchronizedSymbols.scala:140)
at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anonfun$typeParams$1.apply(SynchronizedSymbols.scala:133)
at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$class.gilSynchronizedIfNotThreadsafe(SynchronizedSymbols.scala:123)
at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$8.gilSynchronizedIfNotThreadsafe(SynchronizedSymbols.scala:168)
at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$class.typeParams(SynchronizedSymbols.scala:132)
at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$8.typeParams(SynchronizedSymbols.scala:168)
at scala.reflect.internal.Types$NoArgsTypeRef.typeParams(Types.scala:1931)
at scala.reflect.internal.Types$NoArgsTypeRef.isHigherKinded(Types.scala:1930)
at scala.reflect.internal.tpe.TypeComparers$class.isSubType2(TypeComparers.scala:377)
at scala.reflect.internal.tpe.TypeComparers$class.isSubType1(TypeComparers.scala:320)
at scala.reflect.internal.tpe.TypeComparers$class.isSubType(TypeComparers.scala:278)
at scala.reflect.internal.SymbolTable.isSubType(SymbolTable.scala:16)
at scala.reflect.internal.Types$Type.$less$colon$less(Types.scala:784)
at scala.reflect.internal.Types$Type.$less$colon$less(Types.scala:260)
at org.apache.spark.sql.catalyst.ScalaReflection$.isSubtype(ScalaReflection.scala:83)
at org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$optionOfProductType$1.apply$mcZ$sp(ScalaReflection.scala:677)
at org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$optionOfProductType$1.apply(ScalaReflection.scala:676)
at org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$optionOfProductType$1.apply(ScalaReflection.scala:676)
at scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:56)
at org.apache.spark.sql.catalyst.ScalaReflection$class.cleanUpReflectionObjects(ScalaReflection.scala:926)
at org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:49)
at org.apache.spark.sql.catalyst.ScalaReflection$.optionOfProductType(ScalaReflection.scala:675)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:51)
at org.apache.spark.sql.Encoders$.product(Encoders.scala:275)
Similar questions are answered in link1 and link2, but We already build uber jar for project C, so all the classes should be included in the package. locally the unit test that hits this code works fine but why we get the runtime errors?
