1

I am aware that there is Cloud SQL JDBC Socket Factory. However, that requires Application Default Credentials. What I am trying to do is to use a service Account(Which is the JSON secrets file) to authenticate with the Cloud SQL. Can someone please point me in the right direction? Thanks!

Codes from GettingStarted(https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory):

public class GettingStarted2{
    private static final ImmutableSet<String> SYSTEM_DATABASES =
            ImmutableSet.of(
                    // MySQL.
                    "mysql", "information_schema", "performance_schema",
                    // Postgres.
                    "cloudsqladmin", "postgres");

    @Parameter(names = "-v", description = "Verbose logging.")
    private boolean verbose = false;

    private void run() throws IOException, SQLException {
        System.out.println("Checking API credentials.");
        GoogleCredential apiCredentials;
        try {
//            apiCredentials = GoogleCredential.getApplicationDefault();
            apiCredentials = GoogleCredential.fromStream(new FileInputStream("<keypath.json>"))
                    .createScoped(Lists.newArrayList("https://www.googleapis.com/auth/cloud-platform"));
        } catch (IOException e) {
            System.err.println(
                    "Unable to find API credentials. \nPlease run "
                            + "'gcloud auth application-default login' to make credentials available to "
                            + "this application.");
            if (verbose) {
                e.printStackTrace();
            }
            System.exit(-1);
            return;
        }

        SQLAdmin adminApiClient = createAdminApiClient(apiCredentials);
        Optional<List<DatabaseInstance>> instances = askForProject(adminApiClient);
        if (!instances.isPresent()) {
            return;
        }

        if (instances.get().isEmpty()) {
            System.out.println(
                    "This project does not contain any Cloud SQL instances. "
                            + "Please create one using the Cloud Console.");
            return;
        }

        Optional<DatabaseInstance> optionalInstance = askForInstance(instances.get());
        if (!optionalInstance.isPresent()) {
            return;
        }

        String instanceConnectionName = optionalInstance.get().getConnectionName();
        Optional<DatabaseCredentials> optionalDatabaseCredentials =
                askForDatabaseCredentials(optionalInstance.get());
        if (!optionalDatabaseCredentials.isPresent()) {
            return;
        }

        Connection connection = optionalDatabaseCredentials.get().getConnection();
        List<String> databases = listDatabases(optionalInstance.get(), connection);
        connection.close();
        if (databases.isEmpty()) {
            printConnectionDetails(
                    optionalInstance.get(), Optional.empty(), optionalDatabaseCredentials.get());
            return;
        }

        Optional<String> database = askForDatabase(databases);
        if (!database.isPresent()) {
            return;
        }

        printConnectionDetails(optionalInstance.get(), database, optionalDatabaseCredentials.get());
    }

    private List<String> listDatabases(
            DatabaseInstance databaseInstance, Connection connection) throws SQLException {
        String listDatabasesQuery;
        switch (getDatabaseType(databaseInstance)) {
            case MYSQL:
                listDatabasesQuery = "SHOW DATABASES";
                break;
            case POSTGRES:
                listDatabasesQuery =
                        "SELECT datname AS database FROM pg_database WHERE datistemplate = false";
                break;
            default:
                throw new IllegalStateException();
        }

        Statement statement = connection.createStatement();
        ResultSet resultSet = statement.executeQuery(listDatabasesQuery);
        List<String> databases = new ArrayList<>();
        while (resultSet.next()) {
            String database = resultSet.getString("database");
            if (SYSTEM_DATABASES.contains(database)) {
                continue;
            }
            databases.add(database);
        }
        statement.close();
        databases.sort(String::compareTo);
        return databases;
    }

    private Optional<DatabaseCredentials> askForDatabaseCredentials(DatabaseInstance databaseInstance)
            throws SQLException {

        String defaultUser;
        String displayDatabaseType;
        String defaultDatabase;
        switch (getDatabaseType(databaseInstance)) {
            case MYSQL:
                defaultUser = "root";
                displayDatabaseType = "MySQL";
                defaultDatabase = "mysql";
                break;
            case POSTGRES:
                defaultUser = "postgres";
                displayDatabaseType = "Postgres";
                defaultDatabase = "postgres";
                break;
            default:
                return Optional.empty();
        }

        Console console = System.console();
        String user;
        String lastUser = defaultUser;
        for (; ; ) {
            char[] password={'r','o','o','t'};
            System.out.printf("Please enter %s username [%s]: ", displayDatabaseType, lastUser);
//            user = console.readLine();
            user="root";
            if (user == null) {
                return Optional.empty();
            }

            if (user.trim().isEmpty()) {
                user = lastUser;
            } else {
                lastUser = user;
            }

            System.out.printf("Please enter %s password: ", displayDatabaseType);
            //password = console.readPassword();
            if (password == null) {
                return Optional.empty();
            }

            try {
                return Optional.of(
                        new DatabaseCredentials(
                                user,
                                password,
                                DriverManager.getConnection(
                                        constructJdbcUrl(databaseInstance, defaultDatabase),
                                        user,
                                        new String(password))));
            } catch (SQLException e) {
                if (e.getErrorCode() == 1045) {
                    System.out.println("Invalid username/password. Please try again.");
                    continue;
                }
                // Too bad Postgres doesn't set the error code...
                if (e instanceof PSQLException
                        && e.getMessage().contains("password authentication failed")) {
                    System.out.println("Invalid username/password. Please try again.");
                    continue;
                }
                throw e;
            }
        }
    }

    private static String constructJdbcUrl(DatabaseInstance databaseInstance, String database) {
        switch (getDatabaseType(databaseInstance)) {
            case MYSQL:
                return String.format(
                        "jdbc:mysql://google/%s?socketFactory=com.google.cloud.sql.mysql.SocketFactory" +
                                "&cloudSqlInstance=%s",
                        database,
                        databaseInstance.getConnectionName());
            case POSTGRES:
                return String.format(
                        "jdbc:postgresql://google/%s?socketFactory=com.google.cloud.sql.postgres.SocketFactory" +
                                "&socketFactoryArg=%s",
                        database,
                        databaseInstance.getConnectionName());
            default:
                throw new IllegalStateException();
        }
    }

    private Optional<DatabaseInstance> askForInstance(List<DatabaseInstance> instances) {
        Optional<Integer> instanceChoice =
                chooseFromList(
                        "Please enter the number of the instance you want to use [1]: ",
                        instances.stream()
                                .map(
                                        inst ->
                                                String.format(
                                                        "%s [%s] (%s)",
                                                        inst.getName(),
                                                        inst.getDatabaseVersion(),
                                                        inst.getConnectionName()))
                                .collect(Collectors.toList()));
        if (!instanceChoice.isPresent()) {
            return Optional.empty();
        }

        return Optional.of(instances.get(instanceChoice.get()));
    }

    private Optional<String> askForDatabase(List<String> databases) {
        Optional<Integer> databaseIndex =
                chooseFromList("Please enter the number of the database you want to use [1]: ", databases);
        if (!databaseIndex.isPresent()) {
            return Optional.empty();
        }

        return Optional.of(databases.get(databaseIndex.get()));
    }

    private Optional<Integer> chooseFromList(String prompt, List<String> options) {
        Console console = System.console();

        for (int i = 0; i < options.size(); i++) {
            System.out.println(String.format("%d: %s", i + 1, options.get(i)));
        }

        int choice;
        for (;;) {
            System.out.print(prompt);
//            String line = console.readLine();
            String line="1";
            if (line == null) {
                return Optional.empty();
            }

            if (line.trim().isEmpty()) {
                return Optional.of(0);
            } else {
                try {
                    choice = Integer.parseInt(line);
                } catch (NumberFormatException e) {
                    System.out.println("Invalid choice.");
                    continue;
                }

                if (choice < 1 || choice > options.size()) {
                    System.out.println("Invalid choice.");
                    continue;
                }

                return Optional.of(choice - 1);
            }
        }
    }

    private Optional<List<DatabaseInstance>> askForProject(SQLAdmin adminApiClient)
            throws IOException {
        Console console = System.console();

        InstancesListResponse instancesListResponse = null;
        while (instancesListResponse == null) {
            String project = "";
            while (project.isEmpty()) {
                System.out.print("Enter the name of your Cloud project: ");
                //project = console.readLine();
                project="<projectname>";
                if (project == null) {
                    return Optional.empty();
                }
                project = project.trim();
            }

            System.out.println("Listing Cloud SQL instances.");

            try {
                instancesListResponse = adminApiClient.instances().list(project).execute();
            } catch (GoogleJsonResponseException e) {
                if (e.getStatusCode() >= 400 && e.getStatusCode() < 500) {
                    System.out.println("That doesn't appear to be a valid project, try again.");
                    continue;
                }
                throw e;
            }
        }

        ArrayList<DatabaseInstance> instances = new ArrayList<>(instancesListResponse.getItems());
        instances.sort(Comparator.comparing(DatabaseInstance::getName));
        return Optional.of(instances);
    }

    private void printConnectionDetails(
            DatabaseInstance databaseInstance,
            Optional<String> database,
            DatabaseCredentials databaseCredentials) {
        String databaseName = database.orElse("<database_name>");

        System.out.println("\n\n");
        System.out.printf(
                "Use the following JDBC URL%s:\n\n    %s\n",
                !database.isPresent() ? " after creating a database" : "",
                constructJdbcUrl(databaseInstance, databaseName));
        System.out.println();
        System.out.println("    Username: " + databaseCredentials.getUsername());
        System.out.println(
                "    Password: " + (databaseCredentials.getPassword().length > 0 ? "<yes>" : "<empty>"));
        System.out.println("\n\n");
    }

    private static SQLAdmin createAdminApiClient(Credential credential) {
        HttpTransport httpTransport;
        try {
            httpTransport = GoogleNetHttpTransport.newTrustedTransport();
        } catch (GeneralSecurityException | IOException e) {
            throw new RuntimeException("Unable to initialize HTTP transport", e);
        }

        return new SQLAdmin.Builder(httpTransport, JacksonFactory.getDefaultInstance(), credential)
                .setApplicationName("Cloud SQL Example")
                .build();
    }

    private static final class DatabaseCredentials {
        private final String username;
        private final char[] password;
        private final Connection connection;

        public DatabaseCredentials(String username, char[] password, Connection connection) {
            this.username = username;
            this.password = password;
            this.connection = connection;
        }

        public String getUsername() {
            return username;
        }

        public char[] getPassword() {
            return password;
        }

        public Connection getConnection() {
            return connection;
        }
    }

    private static DatabaseType getDatabaseType(DatabaseInstance databaseInstance) {
        if (databaseInstance.getDatabaseVersion().startsWith("MYSQL_")) {
            return DatabaseType.MYSQL;
        } else if (databaseInstance.getDatabaseVersion().startsWith("POSTGRES_")) {
            return DatabaseType.POSTGRES;
        } else {
            System.err.println("Unsupported database type: " + databaseInstance.getDatabaseVersion());
            System.exit(-1);
            return null;
        }
    }

    private enum DatabaseType {
        MYSQL,
        POSTGRES
    }

    public static void main(String[] args) throws IOException, SQLException {
        GettingStarted2 gettingStarted2 = new GettingStarted2();
        new JCommander(gettingStarted2, args);
        gettingStarted2.run();
    }
}

StackTrace: https://pastebin.com/PMJsAFaK 2nd StackTrace(Using codes from Github):https://pastebin.com/aH5vkLYF

zzdhxu
  • 379
  • 6
  • 22
  • Did you generate the [JSON file correctly](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)? And is the JSON file in the same folder as the `pom.xml` file? Try putting it in there, instead of the "resources package" (What do you mean by that?) When you create the `FileInputStream` object, are you actualling putting the brackets in the string? You shouldn't. As I mentioned in other comment, please do this while running the 'getting-started' example. – Mangu Jun 19 '18 at 15:00
  • Regarding resource packages, it is a package in my project. As for the FileInputStream I did not put the bracket in the string. It is more similar to this: "src/Resources/XXX.json" – zzdhxu Jun 19 '18 at 15:33
  • Added new Stacktrace running using codes from Github. It seems like I am getting a slightly different error at the end, but from what I see they are still searching for the application-default credentials. Appreciate your help! – zzdhxu Jun 19 '18 at 16:02
  • Try to put it where I mentioned several times before, at least as a proof of concept, to discard any user problems. If you still can't run it, you could post what you have on a Github repository of your own, sanitising any possible information before of course, to see if I can run it. If I can run it, I may say it's an issue in your end, but let's see. Also, looking at your last trace, have you modified the code in anyway? (Besides the JSON bit). – Mangu Jun 20 '18 at 09:39
  • Yes. I did place it with the pom.xml file as you mentioned in the latest test. I did not make any changes to the codes. Lastly, looking at the issues opened on [Github](https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory/issues/83) I think it is currently not possible to create a GoogleCredential using the service account and pass it to the factory. Correct me if I am wrong – zzdhxu Jun 20 '18 at 09:48
  • I could make it work with the credentials downloaded as how I mentioned. I tested this before posting, of course. I would suggest that there is a configuration issue here. – Mangu Jun 20 '18 at 09:56
  • Hmm. I see. There is no difference in the codes as the one in the answer though. I have also tried to regenerate the JSON service account but to no avail. Only when I login using the Cloud SDK shell "gcloud auth application-default login" will the codes that run. It only works after I authenticate with the [shell](https://pastebin.com/TPZA7Ypy) – zzdhxu Jun 20 '18 at 10:10
  • Just curious you also used the exact same codes from Github but only changed the GoogleCredentials object and it works? If I am logged in through the Cloud SDK shell it works too. Otherwise it dosen't work for me. Thanks! – zzdhxu Jun 20 '18 at 16:43
  • @Mangu I have attached the updated answer below that I found on their Github issues. Thank you so much for your help! – zzdhxu Jul 12 '18 at 14:50

2 Answers2

1

Update(Answer from Google Engineer): After doing more research on whether it is possible to authenticate using a JSON file instead, here is what I found. They have stated that it currently does not allow External GoogleCredentials authentication.

Currently the way to load ServiceAccount would be this: The factory uses the ApplicationDefaultCredentials to authenticate as you mentioned. You can set these to a service account (it's the first example on that page).

$ gcloud iam service-accounts create my-account
$ gcloud iam service-accounts keys create key.json --iam-account=my-account@my-project.iam.gserviceaccount.com
$ export GOOGLE_APPLICATION_CREDENTIALS=key.json
$ ./my_application.sh

Current open issue(They may add authentication using GoogleCredentials in the future): https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory/issues/66

zzdhxu
  • 379
  • 6
  • 22
0

You can manually provide the credentials when authenticating.

If you follow the examples for the JDBC Socket Factory, in particular this example, you can see that almost everything is set up in the run(). Bearing in mind my first link, you would have to change it to something like this:

GoogleCredential apiCredentials;
    try {
        apiCredentials = GoogleCredential.fromStream(new FileInputStream("yourkey.json"))
    .createScoped(Lists.newArrayList("https://www.googleapis.com/auth/cloud-platform"));

If you follow the example I have linked, don't forget to add the import java.io.FileInputStream and com.google.common.collect.Lists.

Mangu
  • 3,160
  • 2
  • 25
  • 42
  • Hmm. Thanks for your help, but it dosen't really work for me. I hardcoded the other values like my project, user, password etc. but it still comes up with an error as it is still trying to detect the application default credentials. – zzdhxu Jun 19 '18 at 02:38
  • From my debugging even after changing to read from my JSON service account, after I re-authenticate my application-default credentials will the code then run. – zzdhxu Jun 19 '18 at 03:08
  • So, at the moment it runs but not with the credentials you wanted, but with the credentials by default, right? You have mentioned that it didn't work, and comes up with an error. Could you edit your question to include what you have tried (sanitising any sensitive information, of course) and the stacktrace of the error? I may be able to help you with that. – Mangu Jun 19 '18 at 07:31
  • I have added my GettingStarted codes in and the stack trace in pastebin as there was a 30,000 character limit. Hope you can see the stack trace. I hardcoded the values as I am also unsure why it is not prompting me to input the values and will cause a nullPointerException if you try to just run the codes from Github. Thanks for your help! – zzdhxu Jun 19 '18 at 07:51
  • I have been running this sample again and it works for both my MySQL and PostgreSQL instances, and I have been playing with the credentials bit also, as the error is related to it. One quick question before continuing looking into this: Where do you have the JSON file with your credentials located? It should be in the same folder as your 'pom.xml' file. Also, check that the file was generated correctly, you can [generate another one](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) if needed. – Mangu Jun 19 '18 at 10:13
  • I have it in a resources package though and I did not generate a 'pom.xml' file – zzdhxu Jun 19 '18 at 10:21
  • I was also looking at this example. Link attached below. Seems like they did not have any parameters to furnish the service account key. I suppose something goes on in the backend whereby once I call it will start to search for the application-default credentials? https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory/blob/master/examples/compute-engine/src/main/java/com/google/cloud/sql/mysql/example/ListTables.java – zzdhxu Jun 19 '18 at 10:39
  • I'm with Google Cloud Platform Support, and I'm asking other peers to see what can be happening. Let's see if I can find something, but, on a best effort basis, I can't guarantee anything. But, what do you mean by 'I didn't generate a `pom.xml` file'? You are following the [getting-started example](https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory/tree/master/examples/getting-started), right? The `pom.xml` file is already provided, and it works out of the box. – Mangu Jun 19 '18 at 14:10
  • Ok sure. Thank you so much for your help! :) With regards to the pom.xml file I actually copied the codes and run it in my own project, and everything works fine except it still tries to detect application-default credentials. Should I pull the whole project from github and test it instead? Thanks! – zzdhxu Jun 19 '18 at 14:10
  • You can try to discard any possibilities, maybe there are some dependencies missing, but I don't think it would fail on that, as the error should be more related to that. But it doesn't hurt to try and run the example as it is. – Mangu Jun 19 '18 at 14:23
  • Ok sure will try that and update again. Thanks! Hope to hear from you soon too :) – zzdhxu Jun 19 '18 at 14:28
  • @Mangu Even though I have set the application default credentials, I'm not able to connect via JDBC in the main function of a Cloud Dataflow job. I'm receiving connection refused errors in Cloud SQL PostgreSQL. I'm setting GOOGLE_APPLICATION_CREDENTIALS in a environment variable. I can connect to it via JdbcIO though. Do you have any idea why? – d4nielfr4nco Nov 16 '18 at 20:05