Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Catch sqlite exceptions when filter entities #6601

Merged
merged 6 commits into from
Feb 21, 2025
Merged

Catch sqlite exceptions when filter entities #6601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1810c8d
Added failing tests
grzesiek2010 Feb 11, 2025
565f8c2
Catch sqlite exceptions when filter entities
grzesiek2010 Feb 11, 2025
ca26c40
Added a test to cover non existing properties
grzesiek2010 Feb 13, 2025
350be92
Improved tests
grzesiek2010 Feb 15, 2025
6216dd6
Intoduced an EntitiesRepository specyfic exception
grzesiek2010 Feb 17, 2025
eb66fa8
Improved tests
grzesiek2010 Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 34 additions & 28 deletions ...app/src/main/java/org/odk/collect/android/database/entities/DatabaseEntitiesRepository.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import android.content.ContentValues
import android.content.Context
import android.database.Cursor
import android.database.sqlite.SQLiteDatabase
import android.database.sqlite.SQLiteException
import android.provider.BaseColumns._ID
import org.odk.collect.db.sqlite.CursorExt.first
import org.odk.collect.db.sqlite.CursorExt.foldAndClose
Expand All @@ -22,6 +23,7 @@ import org.odk.collect.db.sqlite.toSql
import org.odk.collect.entities.javarosa.parse.EntitySchema
import org.odk.collect.entities.storage.EntitiesRepository
import org.odk.collect.entities.storage.Entity
import org.odk.collect.entities.storage.QueryException
import org.odk.collect.shared.Query
import org.odk.collect.shared.mapColumns

Expand Down Expand Up @@ -258,35 +260,39 @@ class DatabaseEntitiesRepository(context: Context, dbPath: String) : EntitiesRep
}

private fun queryWithAttachedRowId(list: String, query: Query?): List<Entity.Saved> {
return if (query == null) {
databaseConnection.withConnection {
readableDatabase
.rawQuery(
"""
SELECT *, i.$ROW_ID
FROM "$list" e, "${getRowIdTableName(list)}" i
WHERE e._id = i._id
ORDER BY i.$ROW_ID
""".trimIndent(),
null
)
}
} else {
databaseConnection.withConnection {
val sqlQuery = query.toSql()
readableDatabase
.rawQuery(
"""
SELECT *, i.$ROW_ID
FROM "$list" e, "${getRowIdTableName(list)}" i
WHERE e._id = i._id AND ${sqlQuery.selection}
ORDER BY i.$ROW_ID
""".trimIndent(),
sqlQuery.selectionArgs
)
try {
return if (query == null) {
databaseConnection.withConnection {
readableDatabase
.rawQuery(
"""
SELECT *, i.$ROW_ID
FROM "$list" e, "${getRowIdTableName(list)}" i
WHERE e._id = i._id
ORDER BY i.$ROW_ID
""".trimIndent(),
null
)
}
} else {
databaseConnection.withConnection {
val sqlQuery = query.toSql()
readableDatabase
.rawQuery(
"""
SELECT *, i.$ROW_ID
FROM "$list" e, "${getRowIdTableName(list)}" i
WHERE e._id = i._id AND ${sqlQuery.selection}
ORDER BY i.$ROW_ID
""".trimIndent(),
sqlQuery.selectionArgs
)
}
}.foldAndClose {
mapCursorRowToEntity(it, it.getInt(ROW_ID))
}
}.foldAndClose {
mapCursorRowToEntity(it, it.getInt(ROW_ID))
} catch (e: SQLiteException) {
throw QueryException(e.message)
}
}

Expand Down
9 changes: 9 additions & 0 deletions collect_app/src/test/java/org/odk/collect/android/entities/EntitiesRepositoryTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import org.junit.Test
import org.odk.collect.android.entities.support.EntitySameAsMatcher.Companion.sameEntityAs
import org.odk.collect.entities.storage.EntitiesRepository
import org.odk.collect.entities.storage.Entity
import org.odk.collect.entities.storage.QueryException
import org.odk.collect.shared.Query

abstract class EntitiesRepositoryTest {
Expand Down Expand Up @@ -872,4 +873,12 @@ abstract class EntitiesRepositoryTest {
val queriedCanet = repository.query("other.favourite.wines", Query.Eq("label", "Pontet-Canet 2014"))
assertThat(queriedCanet, containsInAnyOrder(sameEntityAs(canet)))
}

@Test(expected = QueryException::class)
fun `#query throws an exception when not existing property is used`() {
val repository = buildSubject()
repository.save("wines", Entity.New("1", "Léoville Barton 2008",))

repository.query("wines", Query.Eq("score", "92"))
}
}
7 changes: 6 additions & 1 deletion ...ies/src/main/java/org/odk/collect/entities/javarosa/filter/LocalEntitiesFilterStrategy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import org.javarosa.xpath.expr.XPathExpression
import org.odk.collect.entities.javarosa.intance.LocalEntitiesInstanceAdapter
import org.odk.collect.entities.javarosa.intance.LocalEntitiesInstanceProvider
import org.odk.collect.entities.storage.EntitiesRepository
import org.odk.collect.entities.storage.QueryException
import org.odk.collect.shared.Query
import java.util.function.Supplier

Expand Down Expand Up @@ -41,7 +42,11 @@ class LocalEntitiesFilterStrategy(entitiesRepository: EntitiesRepository) :
val query = xPathExpressionToQuery(predicate, sourceInstance, evaluationContext)

return if (query != null) {
queryToTreeReferences(query, sourceInstance)
try {
queryToTreeReferences(query, sourceInstance)
} catch (e: QueryException) {
next.get()
}
} else {
next.get()
}
Expand Down
4 changes: 2 additions & 2 deletions entities/src/main/java/org/odk/collect/entities/storage/InMemEntitiesRepository.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class InMemEntitiesRepository : EntitiesRepository {
EntitySchema.VERSION -> it.version.toString()
else -> it.properties.find { propertyName ->
propertyName.first == query.column
}?.second
}?.second ?: throw QueryException("No such column: ${query.column}")
}
fieldName == query.value
}
Expand All @@ -69,7 +69,7 @@ class InMemEntitiesRepository : EntitiesRepository {
EntitySchema.VERSION -> it.version.toString()
else -> it.properties.find { propertyName ->
propertyName.first == query.column
}?.second
}?.second ?: throw QueryException("No such column: ${query.column}")
}
fieldName != query.value
}
Expand Down
3 changes: 3 additions & 0 deletions entities/src/main/java/org/odk/collect/entities/storage/QueryException.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package org.odk.collect.entities.storage

class QueryException(message: String?) : RuntimeException(message)
160 changes: 160 additions & 0 deletions ...src/test/java/org/odk/collect/entities/javarosa/filter/LocalEntitiesFilterStrategyTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,166 @@ class LocalEntitiesFilterStrategyTest {
assertThat(fallthroughFilterStrategy.fellThrough, equalTo(false))
assertThat(instanceProvider.fullParsePerformed, equalTo(false))
}

@Test
fun `works correctly but not in the optimized way with unanswered question = ''`() {
entitiesRepository.save("things", Entity.New("thing1", "Thing1"))

val scenario = Scenario.init(
"Secondary instance form",
html(
head(
title("Secondary instance form"),
model(
mainInstance(
t(
"data id=\"create-entity-form\"",
t("ref_question"),
t("question")
)
),
t("instance id=\"things\" src=\"jr://file-csv/things.csv\""),
bind("/data/ref_question").type("string"),
bind("/data/question").type("string")
)
),
body(
input("/data/ref_question"),
select1Dynamic(
"/data/question",
"instance('things')/root/item[/data/ref_question='']",
"name",
"label"
)
)
),
controllerSupplier
)

val choices = scenario.choicesOf("/data/question").map { it.value }
assertThat(choices, containsInAnyOrder("thing1"))

assertThat(fallthroughFilterStrategy.fellThrough, equalTo(true))
}

@Test
fun `works correctly but not in the optimized way with answered question = value`() {
entitiesRepository.save("things", Entity.New("thing1", "Thing1"))

val scenario = Scenario.init(
"Secondary instance form",
html(
head(
title("Secondary instance form"),
model(
mainInstance(
t(
"data id=\"create-entity-form\"",
t("ref_question"),
t("question")
)
),
t("instance id=\"things\" src=\"jr://file-csv/things.csv\""),
bind("/data/ref_question").type("string"),
bind("/data/question").type("string")
)
),
body(
input("/data/ref_question"),
select1Dynamic(
"/data/question",
"instance('things')/root/item[/data/ref_question='']",
"name",
"label"
)
)
),
controllerSupplier
)
scenario.next()
scenario.answer("blah")

val choices = scenario.choicesOf("/data/question").map { it.value }
assertThat(choices.isEmpty(), equalTo(true))

assertThat(fallthroughFilterStrategy.fellThrough, equalTo(true))
}

@Test
fun `works correctly but not in the optimized way with non existing property = ''`() {
entitiesRepository.save("things", Entity.New("thing1", "Thing1"))

val scenario = Scenario.init(
"Secondary instance form",
html(
head(
title("Secondary instance form"),
model(
mainInstance(
t(
"data id=\"create-entity-form\"",
t("question")
)
),
t("instance id=\"things\" src=\"jr://file-csv/things.csv\""),
bind("/data/question").type("string")
)
),
body(
select1Dynamic(
"/data/question",
"instance('things')/root/item[not_existing_property='']",
"name",
"label"
)
)
),
controllerSupplier
)

val choices = scenario.choicesOf("/data/question").map { it.value }
assertThat(choices, containsInAnyOrder("thing1"))

assertThat(fallthroughFilterStrategy.fellThrough, equalTo(true))
}

@Test
fun `works correctly but not in the optimized way with non existing property = value`() {
entitiesRepository.save("things", Entity.New("thing1", "Thing1"))

val scenario = Scenario.init(
"Secondary instance form",
html(
head(
title("Secondary instance form"),
model(
mainInstance(
t(
"data id=\"create-entity-form\"",
t("question")
)
),
t("instance id=\"things\" src=\"jr://file-csv/things.csv\""),
bind("/data/question").type("string")
)
),
body(
select1Dynamic(
"/data/question",
"instance('things')/root/item[not_existing_property='value']",
"name",
"label"
)
)
),
controllerSupplier
)

val choices = scenario.choicesOf("/data/question").map { it.value }
assertThat(choices.isEmpty(), equalTo(true))

assertThat(fallthroughFilterStrategy.fellThrough, equalTo(true))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

works correctly in the optimized way -- what makes it optimized? This looks like it fell through which I thought would be non-optimized.

This test looks identical to https://github.com/getodk/collect/pull/6601/files#diff-4a25b4c0ba0780186c2edabe26d8518d4e3e6a022a482669373e2a71ceac6dfbR585 to me

In this test not_existing_property is the reference in the secondary instance and 'value' is a static value.

In the test above, undefined is the reference in the secondary instance and /data/ref_question is a static value that came from the form.

It's confusing because the order is different and the intended meaning is different! But I believe that from the system's standpoint they should be in the same category. If there's code somewhere that treats them as different we should dig into it, I think.

Copy link
Member Author

@grzesiek2010 grzesiek2010 Feb 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

works correctly in the optimized way -- what makes it optimized? This looks like it fell through which I thought would be non-optimized.

I made a mistake in the title it's not optimized of course. I will fix that.

If you set not_existing_property='value', it will always return an empty list of entities (because not_existing_property does not exist).
However, if you use undefined=/data/ref_question or 'value'=/data/ref_question (because we focus too much on the undefined case), the list won't be empty - as long as the expected value matches the one entered in the question.
Wouldn't it be enough to treat these two cases as distinct and have separate tests for each?

Copy link
Member

@lognaturel lognaturel Feb 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you set not_existing_property='value', it will always return an empty list of entities (because not_existing_property does not exist).

And with not_existing_property='', you should always get all Entities. That's the same as the undefined=/data/ref_question case when /data/ref_question has a blank value.

I sort of see the distinction that you're making, thanks for explaining it again, that was helpful. It doesn't hurt to have a few extra tests and I don't think there's a fundamental misunderstanding here. 👍

'value'=/data/ref_question

This one does definitely feel different from the other two to me. This should be identified as not an expression that can use the optimized filter strategy and bypass before it even gets there (maybe good to double check this?). The other two should be identified as candidates for the filter strategy and then be kicked back to the next one once the column is determined to be unknown.

Copy link
Member Author

@grzesiek2010 grzesiek2010 Feb 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And with not_existing_property='', you should always get all Entities.

Correct.

This one does definitely feel different from the other two to me. This should be identified as not an expression that can use the optimized filter strategy and bypass before it even gets there (maybe good to double check this?). The other two should be identified as candidates for the filter strategy and then be kicked back to the next one once the column is determined to be unknown.

Right, these cases throw an SQLiteException because of the missing columns:

  • undefined=/data/ref_question
  • null=/data/ref_question
  • not_existing_property=""
  • not_existing_property="value"

These cases are identified as not an expression that can use the optimized:

  • 'value'=/data/ref_question
  • ''=/data/ref_question

}
}

private class FallthroughFilterStrategy : FilterStrategy {
Expand Down