Notebook fails in pipeline due to error 6002 , user error

**Notebooks function well when run normally, however they malfunction when utilized in a pipeline. The cause of the error is data write in csv, however the reason for the failure is unknown. **

Operation on target Notebook 3 failed: ---------------------------------------------------------------------------

Py4JJavaError Traceback (most recent call last)

Cell In[7], line 1

----> 1 df = spark.read.load('abfss://files@storagefortoday.dfs.core.windows.net/data/*.csv', format='csv', header=True)

  2 #display(df.limit(10))
```File /opt/spark/python/lib/pyspark.zip/pyspark/sql/readwriter.py:300, in DataFrameReader.load(self, path, format, schema, **options)

298 self.options(**options)

299 if isinstance(path, str):

301 elif path is not None:

302 if type(path) != list:

   1316 command = proto.CALL_COMMAND_NAME +\

   1317     self.command_header +\

   1318     args_command +\

   1319     proto.END_COMMAND_PART

   1321 answer = self.gateway_client.send_command(command)

-> 1322 return_value = get_return_value(

   1323     answer, self.gateway_client, self.target_id, self.name)

   1325 for temp_arg in temp_args:

   1326     if hasattr(temp_arg, "_detach"):

File /opt/spark/python/lib/pyspark.zip/pyspark/errors/exceptions/captured.py:169, in capture_sql_exception.<locals>.deco(*a, **kw)

167 def deco(*a: Any, **kw: Any) -> Any:

168 try:

170 except Py4JJavaError as e:

171 converted = convert_exception(e.java_exception)

324 value = OUTPUT_CONVERTERtype

325 if answer[1] == REFERENCE_TYPE:

327 "An error occurred while calling {0}{1}{2}.\n".

328 format(target_id, ".", name), value)

329 else:

330 raise Py4JError(

331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".

332 format(target_id, ".", name, value))

: java.nio.file.AccessDeniedException: Operation failed: "This request is not authorized to perform this operation using this permission.", 403, GET, https://storagefortoday.dfs.core.windows.net/files?upn=false&resource=filesystem&maxResults=5000&directory=data&timeout=90&recursive=false, AuthorizationPermissionMismatch, "This request is not authorized to perform this operation using this permission. RequestId:da437775-401f-0018-45cc-bb4518000000 Time:2024-06-11T06:53:08.2745986Z"

at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem.checkException(AzureBlobFileSystem.java:1443)

at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem.listStatus(AzureBlobFileSystem.java:516)

at org.apache.hadoop.fs.Globber.listStatus(Globber.java:128)

at org.apache.hadoop.fs.Globber.doGlob(Globber.java:291)

at org.apache.hadoop.fs.Globber.glob(Globber.java:202)

at org.apache.hadoop.fs.FileSystem.globStatus(FileSystem.java:2124)

at org.apache.spark.deploy.SparkHadoopUtil.globPath(SparkHadoopUtil.scala:316)

at org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$checkAndGlobPathIfNecessary$3(DataSource.scala:738)

at org.apache.spark.util.ThreadUtils$.$anonfun$parmap$2(ThreadUtils.scala:393)

at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)

at scala.util.Success.$anonfun$map$1(Try.scala:255)

at scala.util.Success.map(Try.scala:213)

at scala.concurrent.Future.$anonfun$map$1(Future.scala:292)

at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)

at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)

at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)

at java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)

at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)

at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)

at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)

at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)

at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

at org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.completeExecute(AbfsRestOperation.java:231)

at org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.lambda$execute$0(AbfsRestOperation.java:191)

at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation(IOStatisticsBinding.java:464)

at org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.execute(AbfsRestOperation.java:189)

at org.apache.hadoop.fs.azurebfs.services.AbfsClient.listPath(AbfsClient.java:302)

at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.listStatus(AzureBlobFileSystemStore.java:1158)

at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.listStatus(AzureBlobFileSystemStore.java:1128)

at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.listStatus(AzureBlobFileSystemStore.java:1110)

at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem.listStatus(AzureBlobFileSystem.java:513)

... 20 more

**The above error is shown when the pipeline is triggered**