Adding support for DB downgrade

Adding a schema file for handling DB downgrade. This schema file is part of
the backup/restore set, and hence is available on a device with lower app version.

Bug: 37257575
Change-Id: I69c8ef5f28d5209be6e6679412c7459d4eeda5d0

res/raw/downgrade_schema.json

@@ -0,0 +1,20 @@
+{
+  // Note: Comments are not supported in JSON schema, but android parser is lenient.
+
+  // Maximum DB version supported by this schema
+  "version" : 27,
+
+  // Downgrade from 27 to 26. Empty array indicates, the DB is compatible
+  "downgrade_to_26" : [],
+  "downgrade_to_25" : [],
+  "downgrade_to_24" : [],
+  "downgrade_to_23" : [],
+  "downgrade_to_22" : [
+    "ALTER TABLE favorites RENAME TO temp_favorites;",
+    "CREATE TABLE favorites(_id INTEGER PRIMARY KEY, title TEXT, intent TEXT, container INTEGER, screen INTEGER, cellX INTEGER, cellY INTEGER, spanX INTEGER, spanY INTEGER, itemType INTEGER, appWidgetId INTEGER NOT NULL DEFAULT - 1, iconPackage TEXT, iconResource TEXT, icon BLOB, appWidgetProvider TEXT, modified INTEGER NOT NULL DEFAULT 0, restored INTEGER NOT NULL DEFAULT 0, profileId INTEGER DEFAULT 0, rank INTEGER NOT NULL DEFAULT 0);",
+    "INSERT INTO favorites SELECT _id, title, intent, container, screen, cellX, cellY, spanX, spanY, itemType, appWidgetId, iconPackage, iconResource, icon, appWidgetProvider, modified, restored, profileId, rank FROM temp_favorites;",
+    "DROP TABLE temp_favorites;"
+  ]
+
+  // Missing values indicate the DB is not compatible
+}

res/xml/backupscheme.xml

@@ -3,5 +3,6 @@
 
     <include domain="database" path="launcher.db" />
     <include domain="sharedpref" path="com.android.launcher3.prefs.xml" />
+    <include domain="file" path="downgrade_schema.json" />
 
 </full-backup-content>

src/com/android/launcher3/LauncherProvider.java

@@ -56,6 +56,7 @@ import com.android.launcher3.config.FeatureFlags;
 import com.android.launcher3.dynamicui.ExtractionUtils;
 import com.android.launcher3.graphics.IconShapeOverride;
 import com.android.launcher3.logging.FileLog;
+import com.android.launcher3.model.DbDowngradeHelper;
 import com.android.launcher3.provider.LauncherDbUtils;
 import com.android.launcher3.provider.LauncherDbUtils.SQLiteTransaction;
 import com.android.launcher3.provider.RestoreDbTask;
@@ -64,6 +65,7 @@ import com.android.launcher3.util.NoLocaleSqliteContext;
 import com.android.launcher3.util.Preconditions;
 import com.android.launcher3.util.Thunk;
 
+import java.io.File;
 import java.io.FileDescriptor;
 import java.io.PrintWriter;
 import java.lang.reflect.Method;
@@ -77,18 +79,12 @@ public class LauncherProvider extends ContentProvider {
     private static final String TAG = "LauncherProvider";
     private static final boolean LOGD = false;
 
+    private static final String DOWNGRADE_SCHEMA_FILE = "downgrade_schema.json";
+
     /**
      * Represents the schema of the database. Changes in scheme need not be backwards compatible.
      */
-    private static final int SCHEMA_VERSION = 27;
-    /**
-     * Represents the actual data. It could include additional validations and normalizations added
-     * overtime. These must be backwards compatible, else we risk breaking old devices during
-     * restore or binary version downgrade.
-     */
-    private static final int DATA_VERSION = 3;
-
-    private static final String PREF_KEY_DATA_VERISON = "provider_data_version";
+    public static final int SCHEMA_VERSION = 27;
 
     public static final String AUTHORITY = (BuildConfig.APPLICATION_ID + ".settings").intern();
 
@@ -703,47 +699,30 @@ public class LauncherProvider extends ContentProvider {
         @Override
         public void onOpen(SQLiteDatabase db) {
             super.onOpen(db);
-            SharedPreferences prefs = mContext
-                    .getSharedPreferences(LauncherFiles.DEVICE_PREFERENCES_KEY, 0);
-            int oldVersion = prefs.getInt(PREF_KEY_DATA_VERISON, 0);
-            if (oldVersion != DATA_VERSION) {
-                // Only run the data upgrade path for an existing db.
-                if (!Utilities.getPrefs(mContext).getBoolean(EMPTY_DATABASE_CREATED, false)) {
-                    try (SQLiteTransaction t = new SQLiteTransaction(db)) {
-                        onDataUpgrade(db, oldVersion);
-                        t.commit();
-                    } catch (Exception e) {
-                        Log.d(TAG, "Error updating data version, ignoring", e);
-                        return;
-                    }
-                }
-                prefs.edit().putInt(PREF_KEY_DATA_VERISON, DATA_VERSION).apply();
+
+            File schemaFile = mContext.getFileStreamPath(DOWNGRADE_SCHEMA_FILE);
+            if (!schemaFile.exists()) {
+                handleOneTimeDataUpgrade(db);
             }
+            DbDowngradeHelper.updateSchemaFile(schemaFile, SCHEMA_VERSION, mContext,
+                    R.raw.downgrade_schema);
         }
 
         /**
-         * Called when the data is updated as part of app update. It can be called multiple times
-         * with old version, even though it had been run before. The changes made here must be
-         * backwards compatible, else we risk breaking old devices during restore or binary
-         * version downgrade.
+         * One-time data updated before support of onDowngrade was added. This update is backwards
+         * compatible and can safely be run multiple times.
+         * Note: No new logic should be added here after release, as the new logic might not get
+         * executed on an existing device.
+         * TODO: Move this to db upgrade path, once the downgrade path is released.
          */
-        protected void onDataUpgrade(SQLiteDatabase db, int oldVersion) {
-            switch (oldVersion) {
-                case 0:
-                case 1: {
-                    // Remove "profile extra"
-                    UserManagerCompat um = UserManagerCompat.getInstance(mContext);
-                    for (UserHandle user : um.getUserProfiles()) {
-                        long serial = um.getSerialNumberForUser(user);
-                        String sql = "update favorites set intent = replace(intent, "
-                                + "';l.profile=" + serial + ";', ';') where itemType = 0;";
-                        db.execSQL(sql);
-                    }
-                }
-                case 2:
-                case 3:
-                    // data updated
-                    return;
+        protected void handleOneTimeDataUpgrade(SQLiteDatabase db) {
+            // Remove "profile extra"
+            UserManagerCompat um = UserManagerCompat.getInstance(mContext);
+            for (UserHandle user : um.getUserProfiles()) {
+                long serial = um.getSerialNumberForUser(user);
+                String sql = "update favorites set intent = replace(intent, "
+                        + "';l.profile=" + serial + ";', ';') where itemType = 0;";
+                db.execSQL(sql);
             }
         }
 
@@ -850,15 +829,14 @@ public class LauncherProvider extends ContentProvider {
 
         @Override
         public void onDowngrade(SQLiteDatabase db, int oldVersion, int newVersion) {
-            if (oldVersion == 28 && newVersion == 27) {
-                // TODO: remove this check. This is only applicable for internal development/testing
-                // and for any released version of Launcher.
-                return;
+            try {
+                DbDowngradeHelper.parse(mContext.getFileStreamPath(DOWNGRADE_SCHEMA_FILE))
+                        .onDowngrade(db, oldVersion, newVersion);
+            } catch (Exception e) {
+                Log.d(TAG, "Unable to downgrade from: " + oldVersion + " to " + newVersion +
+                        ". Wiping databse.", e);
+                createEmptyDB(db);
             }
-            // This shouldn't happen -- throw our hands up in the air and start over.
-            Log.w(TAG, "Database version downgrade from: " + oldVersion + " to " + newVersion +
-                    ". Wiping databse.");
-            createEmptyDB(db);
         }
 
         /**

src/com/android/launcher3/model/DbDowngradeHelper.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.launcher3.model;
+
+import android.content.Context;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteException;
+import android.util.Log;
+import android.util.SparseArray;
+
+import com.android.launcher3.provider.LauncherDbUtils.SQLiteTransaction;
+import com.android.launcher3.util.IOUtils;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+
+/**
+ * Utility class to handle DB downgrade
+ */
+public class DbDowngradeHelper {
+
+    private static final String TAG = "DbDowngradeHelper";
+
+    private static final String KEY_VERSION = "version";
+    private static final String KEY_DOWNGRADE_TO = "downgrade_to_";
+
+    private final SparseArray<String[]> mStatements = new SparseArray<>();
+    public final int version;
+
+    private DbDowngradeHelper(int version) {
+        this.version = version;
+    }
+
+    public void onDowngrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+        ArrayList<String> allCommands = new ArrayList<>();
+
+        for (int i = oldVersion - 1; i >= newVersion; i--) {
+            String[] commands = mStatements.get(i);
+            if (commands == null) {
+                throw new SQLiteException("Downgrade path not supported to version " + i);
+            }
+            Collections.addAll(allCommands, commands);
+        }
+
+        try (SQLiteTransaction t = new SQLiteTransaction(db)) {
+            for (String sql : allCommands) {
+                db.execSQL(sql);
+            }
+            t.commit();
+        }
+    }
+
+    public static DbDowngradeHelper parse(File file) throws JSONException, IOException {
+        JSONObject obj = new JSONObject(new String(IOUtils.toByteArray(file)));
+        DbDowngradeHelper helper = new DbDowngradeHelper(obj.getInt(KEY_VERSION));
+        for (int version = helper.version - 1; version > 0; version--) {
+            if (obj.has(KEY_DOWNGRADE_TO + version)) {
+                JSONArray statements = obj.getJSONArray(KEY_DOWNGRADE_TO + version);
+                String[] parsed = new String[statements.length()];
+                for (int i = 0; i < parsed.length; i++) {
+                    parsed[i] = statements.getString(i);
+                }
+                helper.mStatements.put(version, parsed);
+            }
+        }
+        return helper;
+    }
+
+    public static void updateSchemaFile(File schemaFile, int expectedVersion,
+            Context context, int schemaResId) {
+        try {
+            if (DbDowngradeHelper.parse(schemaFile).version >= expectedVersion) {
+                return;
+            }
+        } catch (Exception e) {
+            // Schema error
+        }
+
+        // Write the updated schema
+        try (FileOutputStream fos = new FileOutputStream(schemaFile);
+            InputStream in = context.getResources().openRawResource(schemaResId)) {
+            IOUtils.copy(in, fos);
+        } catch (IOException e) {
+            Log.e(TAG, "Error writing schema file", e);
+        }
+    }
+}

src/com/android/launcher3/provider/LauncherDbUtils.java

@@ -52,6 +52,7 @@ public class LauncherDbUtils {
 
             if (screenIds.isEmpty()) {
                 // No update needed
+                t.commit();
                 return true;
             }
             if (screenIds.get(0) != 0) {
@@ -71,6 +72,7 @@ public class LauncherDbUtils {
             if (DatabaseUtils.queryNumEntries(db, Favorites.TABLE_NAME,
                     "container = -100 and screen = 0 and cellY = 0") == 0) {
                 // First row is empty, no need to migrate.
+                t.commit();
                 return true;
             }
 

src/com/android/launcher3/util/IOUtils.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.launcher3.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Supports various IO utility functions
+ */
+public class IOUtils {
+
+    private static final int BUF_SIZE = 0x1000; // 4K
+
+    public static byte[] toByteArray(File file) throws IOException {
+        try (InputStream in = new FileInputStream(file)) {
+            return toByteArray(in);
+        }
+    }
+
+    public static byte[] toByteArray(InputStream in) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        copy(in, out);
+        return out.toByteArray();
+    }
+
+    public static long copy(InputStream from, OutputStream to) throws IOException {
+        byte[] buf = new byte[BUF_SIZE];
+        long total = 0;
+        int r;
+        while ((r = from.read(buf)) != -1) {
+            to.write(buf, 0, r);
+            total += r;
+        }
+        return total;
+    }
+}

tests/res/raw/db_schema_v10.json

@@ -0,0 +1,4 @@
+{
+  "version" : 10,
+  "downgrade_to_9" : []
+}

tests/src/com/android/launcher3/model/DbDowngradeHelperTest.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.launcher3.model;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNotSame;
+import static junit.framework.Assert.assertTrue;
+
+import android.content.ContentValues;
+import android.content.Context;
+import android.database.Cursor;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+import android.support.test.InstrumentationRegistry;
+import android.support.test.filters.SmallTest;
+import android.support.test.runner.AndroidJUnit4;
+
+import com.android.launcher3.LauncherProvider;
+import com.android.launcher3.LauncherProvider.DatabaseHelper;
+import com.android.launcher3.LauncherSettings.Favorites;
+import com.android.launcher3.R;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+
+/**
+ * Tests for {@link DbDowngradeHelper}
+ */
+@SmallTest
+@RunWith(AndroidJUnit4.class)
+public class DbDowngradeHelperTest {
+
+    private static final String SCHEMA_FILE = "test_schema.json";
+    private static final String DB_FILE = "test.db";
+
+    private Context mContext;
+    private File mSchemaFile;
+    private File mDbFile;
+
+    @Before
+    public void setup() {
+        mContext = InstrumentationRegistry.getTargetContext();
+        mSchemaFile = mContext.getFileStreamPath(SCHEMA_FILE);
+        mDbFile = mContext.getDatabasePath(DB_FILE);
+    }
+
+    @Test
+    public void testUpdateSchemaFile() throws Exception {
+        Context myContext = InstrumentationRegistry.getContext();
+        int testResId = myContext.getResources().getIdentifier(
+                "db_schema_v10", "raw", myContext.getPackageName());
+        mSchemaFile.delete();
+        assertFalse(mSchemaFile.exists());
+
+        DbDowngradeHelper.updateSchemaFile(mSchemaFile, 10, myContext, testResId);
+        assertTrue(mSchemaFile.exists());
+        assertEquals(10, DbDowngradeHelper.parse(mSchemaFile).version);
+
+        // Schema is updated on version upgrade
+        assertTrue(mSchemaFile.setLastModified(0));
+        DbDowngradeHelper.updateSchemaFile(mSchemaFile, 11, myContext, testResId);
+        assertNotSame(0, mSchemaFile.lastModified());
+
+        // Schema is not updated when version is same
+        assertTrue(mSchemaFile.setLastModified(0));
+        DbDowngradeHelper.updateSchemaFile(mSchemaFile, 10, myContext, testResId);
+        assertEquals(0, mSchemaFile.lastModified());
+
+        // Schema is not updated on version downgrade
+        DbDowngradeHelper.updateSchemaFile(mSchemaFile, 3, myContext, testResId);
+        assertEquals(0, mSchemaFile.lastModified());
+    }
+
+    @Test
+    public void testDowngrade_success_v24() throws Exception {
+        setupTestDb();
+
+        TestOpenHelper helper = new TestOpenHelper(24);
+        assertEquals(24, helper.getReadableDatabase().getVersion());
+        helper.close();
+    }
+
+    @Test
+    public void testDowngrade_success_v22() throws Exception {
+        setupTestDb();
+
+        SQLiteOpenHelper helper = new TestOpenHelper(22);
+        assertEquals(22, helper.getWritableDatabase().getVersion());
+
+        // Check column does not exist
+        try (Cursor c = helper.getWritableDatabase().query(Favorites.TABLE_NAME,
+                null, null, null, null, null, null)) {
+            assertEquals(-1, c.getColumnIndex(Favorites.OPTIONS));
+
+            // Check data is present
+            assertEquals(10, c.getCount());
+        }
+        helper.close();
+
+        helper = new DatabaseHelper(mContext, null, DB_FILE) {
+            @Override
+            public void onOpen(SQLiteDatabase db) { }
+        };
+        assertEquals(LauncherProvider.SCHEMA_VERSION, helper.getWritableDatabase().getVersion());
+
+        try (Cursor c = helper.getWritableDatabase().query(Favorites.TABLE_NAME,
+                null, null, null, null, null, null)) {
+            // Check column exists
+            assertNotSame(-1, c.getColumnIndex(Favorites.OPTIONS));
+
+            // Check data is present
+            assertEquals(10, c.getCount());
+        }
+        helper.close();
+    }
+
+    @Test(expected = DowngradeFailException.class)
+    public void testDowngrade_fail_v20() throws Exception {
+        setupTestDb();
+
+        TestOpenHelper helper = new TestOpenHelper(20);
+        helper.getReadableDatabase().getVersion();
+    }
+
+    private void setupTestDb() throws Exception {
+        mSchemaFile.delete();
+        mDbFile.delete();
+
+        DbDowngradeHelper.updateSchemaFile(mSchemaFile, LauncherProvider.SCHEMA_VERSION, mContext,
+                R.raw.downgrade_schema);
+
+        DatabaseHelper dbHelper = new DatabaseHelper(mContext, null, DB_FILE) {
+            @Override
+            public void onOpen(SQLiteDatabase db) { }
+        };
+        // Insert dummy data
+        for (int i = 0; i < 10; i++) {
+            ContentValues values = new ContentValues();
+            values.put(Favorites._ID, i);
+            values.put(Favorites.TITLE, "title " + i);
+            dbHelper.getWritableDatabase().insert(Favorites.TABLE_NAME, null, values);
+        }
+        dbHelper.close();
+    }
+
+    private class TestOpenHelper extends SQLiteOpenHelper {
+
+        public TestOpenHelper(int version) {
+            super(mContext, DB_FILE, null, version);
+        }
+
+        @Override
+        public void onCreate(SQLiteDatabase sqLiteDatabase) {
+            throw new RuntimeException("DB should already be created");
+        }
+
+        @Override
+        public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+            throw new RuntimeException("Only downgrade supported");
+        }
+
+        @Override
+        public void onDowngrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+            try {
+                DbDowngradeHelper.parse(mSchemaFile).onDowngrade(db, oldVersion, newVersion);
+            } catch (Exception e) {
+                throw new DowngradeFailException(e);
+            }
+        }
+    }
+
+    private static class DowngradeFailException extends RuntimeException {
+        public DowngradeFailException(Exception e) {
+            super(e);
+        }
+    }
+}