Dataframereader and null pointer exception
0
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
import org.apache.spark.sql.types._
import java.util.Calendar
import java.text.SimpleDateFormat
import org.apache.spark.sql._
import java.sql.Timestamp
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
.load()
-- not the above loads a dataframereader cause of the use of the .option(). Hence show() and printSchema() dont work on a dataframereader but will work if it was a dataframe
SRVY_DF.write.partitionBy("prtn").mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
I am getting the same error , although i have used load.
When i change the code to call load explicitly , it works
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
val df2=SRVY_DF.load()
When i attempt to write it though it gives a java.lang.null pointer exception
df2.write.mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
scala dataframe nullpointerexception
asked Nov 22 at 19:22
S T
63
add a comment |
0
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
import org.apache.spark.sql.types._
import java.util.Calendar
import java.text.SimpleDateFormat
import org.apache.spark.sql._
import java.sql.Timestamp
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
.load()
-- not the above loads a dataframereader cause of the use of the .option(). Hence show() and printSchema() dont work on a dataframereader but will work if it was a dataframe
SRVY_DF.write.partitionBy("prtn").mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
I am getting the same error , although i have used load.
When i change the code to call load explicitly , it works
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
val df2=SRVY_DF.load()
When i attempt to write it though it gives a java.lang.null pointer exception
df2.write.mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
scala dataframe nullpointerexception
asked Nov 22 at 19:22
S T
63
add a comment |
0
0
0
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
import org.apache.spark.sql.types._
import java.util.Calendar
import java.text.SimpleDateFormat
import org.apache.spark.sql._
import java.sql.Timestamp
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
.load()
-- not the above loads a dataframereader cause of the use of the .option(). Hence show() and printSchema() dont work on a dataframereader but will work if it was a dataframe
SRVY_DF.write.partitionBy("prtn").mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
I am getting the same error , although i have used load.
When i change the code to call load explicitly , it works
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
val df2=SRVY_DF.load()
When i attempt to write it though it gives a java.lang.null pointer exception
df2.write.mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
scala dataframe nullpointerexception
asked Nov 22 at 19:22
S T
63
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
import org.apache.spark.sql.types._
import java.util.Calendar
import java.text.SimpleDateFormat
import org.apache.spark.sql._
import java.sql.Timestamp
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
.load()
-- not the above loads a dataframereader cause of the use of the .option(). Hence show() and printSchema() dont work on a dataframereader but will work if it was a dataframe
SRVY_DF.write.partitionBy("prtn").mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
I am getting the same error , although i have used load.
When i change the code to call load explicitly , it works
val SRVY_DF=spark.read.format("jdbc")
.option("url","jdbc:teradata://teradata-dns-sysa.fg.rbc.com")
.option("driver","com.teradata.jdbc.TeraDriver")
.option("dbtable","DDWV21.SRVY")
.option("user","ph9au")
.option("password","H9aoserviceID")
.option("dbtable",s"(SELECT extract (year from CXC.CAL_DT) || cast ((cast(cxc.cal_dt as format 'mmm')) as char(3)) as prtn ,CXC.CAL_DT, CP.* FROM DL_2015_PMD_POC.CALL_PURPOSE CP INNER JOIN DL_2015_PMD_POC.MTRC_CALLXCALL CXC ON CP.INTCN_RSRC_ID=CXC.INTCN_RSRC_ID WHERE CAL_DT<='2016-10-31') as st")
val df2=SRVY_DF.load()
When i attempt to write it though it gives a java.lang.null pointer exception
df2.write.mode(SaveMode.Overwrite).parquet("hdfs:/prod/11323/app/H9A0/data/Call_Purpose/CP.parquet")
scala dataframe nullpointerexception
scala dataframe nullpointerexception
asked Nov 22 at 19:22
S T
63
asked Nov 22 at 19:22
S T
63
asked Nov 22 at 19:22
S T
63
asked Nov 22 at 19:22
S T
63
asked Nov 22 at 19:22
S T
63
63
add a comment |
add a comment |
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53437025%2fdataframereader-and-null-pointer-exception%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
active
oldest
votes
active
oldest
votes
active
oldest
votes
active
oldest
votes
draft saved
draft discarded
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Some of your past answers have not been well-received, and you're in danger of being blocked from answering.
Please pay close attention to the following guidance:
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53437025%2fdataframereader-and-null-pointer-exception%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
