Spark can't pickle method_descriptor

Vote count:

0

I get this weird error message

Traceback (most recent call last):
  File "/home/user/inverted-index.py", line 79, in <module>
    sc.wholeTextFiles(data_dir).flatMap(update).groupByKey().map(store)
  File "/home/user/spark/python/pyspark/rdd.py", line 1650, in groupByKey
    numPartitions).mapValues(lambda x: ResultIterable(x))
  File "/home/user/spark/python/pyspark/rdd.py", line 1564, in combineByKey
    numPartitions = self._defaultReducePartitions()
  File "/home/user/spark/python/pyspark/rdd.py", line 1952, in _defaultReducePartitions
    return self.getNumPartitions()
  File "/home/user/spark/python/pyspark/rdd.py", line 391, in getNumPartitions
    return self._jrdd.partitions().size()
  File "/home/user/spark/python/pyspark/rdd.py", line 2013, in _jrdd
    pickled_command = ser.dumps(command)
  File "/home/user/spark/python/pyspark/serializers.py", line 364, in dumps
    return cloudpickle.dumps(obj, 2)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 802, in dumps
    cp.dump(obj)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 140, in dump
    return pickle.Pickler.dump(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 224, in dump
    self.save(obj)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.7/pickle.py", line 548, in save_tuple
    save(element)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 259, in save_function
    self.save_function_tuple(obj, [themodule])
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 316, in save_function_tuple
    save(closure)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.7/pickle.py", line 600, in save_list
    self._batch_appends(iter(obj))
  File "/usr/lib/python2.7/pickle.py", line 636, in _batch_appends
    save(tmp[0])
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 254, in save_function
    self.save_function_tuple(obj, modList)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 314, in save_function_tuple
    save(f_globals)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 632, in save_reduce
    save(state)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 632, in save_reduce
    save(state)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 632, in save_reduce
    save(state)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 545, in save_inst
    self.save_inst_logic(obj)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 535, in save_inst_logic
    save(stuff)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 545, in save_inst
    self.save_inst_logic(obj)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 535, in save_inst_logic
    save(stuff)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 598, in save_reduce
    save(cls)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 464, in save_global
    d),obj=obj)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 613, in save_reduce
    save(args)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.7/pickle.py", line 548, in save_tuple
    save(element)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 181, in save_dict
    pickle.Pickler.save_dict(self, obj)
  File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
    save(v)
  File "/usr/lib/python2.7/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 598, in save_reduce
    save(cls)
  File "/usr/lib/python2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/user/spark/python/pyspark/cloudpickle.py", line 439, in save_global
    raise pickle.PicklingError("Can't pickle builtin %s" % obj)
pickle.PicklingError: Can't pickle builtin <type 'method_descriptor'>

My update func returns a list of tuples of type (key, (value1, value2)) and all of them are strings

What is going wrong?

asked 44 secs ago

Θεόφιλος Μουρατίδης

731

Spark can't pickle method_descriptor