Compare value of Dataframe column with list value
3
I have a spark dataframe columns 'id' and 'articles' and a list of values 'a_list' as below.
df = spark.createDataFrame([(1, 4), (2, 3), (5, 6)], ("id", "articles"))
a_list = [1, 4, 6]
I am trying to compare list value with value of dataframe column "articles" and if match found updating column 'E' to 1 else 0
I am using "isin" in my code below
df['E'] = df.articles.isin(a_list).astype(int)
Getting
TypeError: unexpected type:
<type 'type'>
What am I missing here ?
python python-3.x pyspark
edited Nov 14 '18 at 0:42
Psidom
123k1285127
asked Nov 14 '18 at 0:40
UmiUmi
626
add a comment |
3
I have a spark dataframe columns 'id' and 'articles' and a list of values 'a_list' as below.
df = spark.createDataFrame([(1, 4), (2, 3), (5, 6)], ("id", "articles"))
a_list = [1, 4, 6]
I am trying to compare list value with value of dataframe column "articles" and if match found updating column 'E' to 1 else 0
I am using "isin" in my code below
df['E'] = df.articles.isin(a_list).astype(int)
Getting
TypeError: unexpected type:
<type 'type'>
What am I missing here ?
python python-3.x pyspark
edited Nov 14 '18 at 0:42
Psidom
123k1285127
asked Nov 14 '18 at 0:40
UmiUmi
626
add a comment |
3
3
3
I have a spark dataframe columns 'id' and 'articles' and a list of values 'a_list' as below.
df = spark.createDataFrame([(1, 4), (2, 3), (5, 6)], ("id", "articles"))
a_list = [1, 4, 6]
I am trying to compare list value with value of dataframe column "articles" and if match found updating column 'E' to 1 else 0
I am using "isin" in my code below
df['E'] = df.articles.isin(a_list).astype(int)
Getting
TypeError: unexpected type:
<type 'type'>
What am I missing here ?
python python-3.x pyspark
edited Nov 14 '18 at 0:42
Psidom
123k1285127
asked Nov 14 '18 at 0:40
UmiUmi
626
I have a spark dataframe columns 'id' and 'articles' and a list of values 'a_list' as below.
df = spark.createDataFrame([(1, 4), (2, 3), (5, 6)], ("id", "articles"))
a_list = [1, 4, 6]
I am trying to compare list value with value of dataframe column "articles" and if match found updating column 'E' to 1 else 0
I am using "isin" in my code below
df['E'] = df.articles.isin(a_list).astype(int)
Getting
TypeError: unexpected type:
<type 'type'>
What am I missing here ?
python python-3.x pyspark
python python-3.x pyspark
edited Nov 14 '18 at 0:42
Psidom
123k1285127
asked Nov 14 '18 at 0:40
UmiUmi
626
edited Nov 14 '18 at 0:42
Psidom
123k1285127
asked Nov 14 '18 at 0:40
UmiUmi
626
edited Nov 14 '18 at 0:42
Psidom
123k1285127
edited Nov 14 '18 at 0:42
Psidom
123k1285127
edited Nov 14 '18 at 0:42
Psidom
123k1285127
123k1285127
asked Nov 14 '18 at 0:40
UmiUmi
626
asked Nov 14 '18 at 0:40
UmiUmi
626
asked Nov 14 '18 at 0:40
UmiUmi
626
626
add a comment |
add a comment |
1 Answer
1
active
oldest
votes
2
Provide your type as string "int" instead of int which is python's native type that spark doesn't recognize; Also to create a column in spark data frame, use withColumn method instead of direct assignment:
df.withColumn('E', df.articles.isin(a_list).astype('int')).show()
+---+--------+---+
| id|articles| E|
+---+--------+---+
| 1| 4| 1|
| 2| 3| 0|
| 5| 6| 1|
+---+--------+---+
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
1
You can usewhen.otherwiseto conditionally create a column.import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))
– Psidom
Nov 14 '18 at 1:00
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53291537%2fcompare-value-of-dataframe-column-with-list-value%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
2
Provide your type as string "int" instead of int which is python's native type that spark doesn't recognize; Also to create a column in spark data frame, use withColumn method instead of direct assignment:
df.withColumn('E', df.articles.isin(a_list).astype('int')).show()
+---+--------+---+
| id|articles| E|
+---+--------+---+
| 1| 4| 1|
| 2| 3| 0|
| 5| 6| 1|
+---+--------+---+
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
1
You can usewhen.otherwiseto conditionally create a column.import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))
– Psidom
Nov 14 '18 at 1:00
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
add a comment |
2
Provide your type as string "int" instead of int which is python's native type that spark doesn't recognize; Also to create a column in spark data frame, use withColumn method instead of direct assignment:
df.withColumn('E', df.articles.isin(a_list).astype('int')).show()
+---+--------+---+
| id|articles| E|
+---+--------+---+
| 1| 4| 1|
| 2| 3| 0|
| 5| 6| 1|
+---+--------+---+
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
1
You can usewhen.otherwiseto conditionally create a column.import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))
– Psidom
Nov 14 '18 at 1:00
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
add a comment |
2
2
2
Provide your type as string "int" instead of int which is python's native type that spark doesn't recognize; Also to create a column in spark data frame, use withColumn method instead of direct assignment:
df.withColumn('E', df.articles.isin(a_list).astype('int')).show()
+---+--------+---+
| id|articles| E|
+---+--------+---+
| 1| 4| 1|
| 2| 3| 0|
| 5| 6| 1|
+---+--------+---+
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
Provide your type as string "int" instead of int which is python's native type that spark doesn't recognize; Also to create a column in spark data frame, use withColumn method instead of direct assignment:
df.withColumn('E', df.articles.isin(a_list).astype('int')).show()
+---+--------+---+
| id|articles| E|
+---+--------+---+
| 1| 4| 1|
| 2| 3| 0|
| 5| 6| 1|
+---+--------+---+
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
answered Nov 14 '18 at 0:46
PsidomPsidom
123k1285127
123k1285127
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
1
You can usewhen.otherwiseto conditionally create a column.import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))
– Psidom
Nov 14 '18 at 1:00
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
add a comment |
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
1
You can usewhen.otherwiseto conditionally create a column.import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))
– Psidom
Nov 14 '18 at 1:00
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
Qq , so instead of updating column with int type , if i were to update it with a text , for 1= Confirmed and for 0 Not confirmed , is there way to do it within above solution ?
– Umi
Nov 14 '18 at 0:53
1
1
You can use
when.otherwise to conditionally create a column. import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))– Psidom
Nov 14 '18 at 1:00
You can use
when.otherwise to conditionally create a column. import pyspark.sql.functions as f; df.withColumn('E', f.when(df.articles.isin(a_list), 'confirmed').otherwise('not confirmed'))– Psidom
Nov 14 '18 at 1:00
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
Sorry to bother again ,if i have a mutiple list like a_list=[4, 10] , b_list=[11,6] , c_list=[3,4] . How can i check against each list and if match found , update column "E" to "Found in a_list" (if found in a_list) or "Found in b_list" (if found in b_list) or "Found in b_list" (if found in c_list)
– Umi
Nov 14 '18 at 1:14
add a comment |
draft saved
draft discarded
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53291537%2fcompare-value-of-dataframe-column-with-list-value%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
