Hello,
We are migrating data from Oracle or SQL Server to a new database design in MySQL, Oracle or SQL Server (whichever the customer is using). We have certain user-definable columns that exist in every table. We are moving these columns to rows in a standard table that contains all user defined columns. I would like to create a generic job/transformation(s) that runs a query against the database (will be different for Oracle/SQL Server) that will create a query with all the columns for the table that need to be brought over (so I don't have to write a query for every table in our database which is over 1000 tables). The output from the "table input" will be multiple rows containing the SQL query. I want to use this output as the SQL for the next "table input". I'm having a hard time figuring out how to do this. I was able to use the SQL Server "stuff" command to concatenate all rows together into a single column and assign that "field" to a "variable". Well, I cannot use this solution for Oracle because the generated column can be over 4,000 characters and all the options for oracle to put multiple rows to 1 column do not work (my query result gets truncated). So, I need a way to take the multiple row output of a "table input" step and use that as the "SQL" portion of another "table input".
Here's the sample query in SQL Server sytax ('NONCONFORMANCE' will be replaced with a variable indicating table):
SELECT 'SELECT '''+COLUMN_NAME+''' AS REFERENCE_TYPE, '+
CASE WHEN DATA_TYPE IN ('varchar','CHAR','NVARCHAR','NCHAR') THEN COLUMN_NAME
WHEN DATA_TYPE IN ('datetime','date') THEN 'CONVERT(varchar,'+COLUMN_NAME+',109)'
ELSE 'CONVERT(varchar,'+COLUMN_NAME+')' END+
' AS textValue, b.SYSID AS XREF_SYSID'+
' FROM '+TABLE_NAME+' a INNER JOIN XREF_TABLE b ON a.'+PK+' = b.PK AND b.TABLE_NAME = '''+a.TABLE_NAME+''' WHERE '+COLUMN_NAME+' IS NOT NULL'+
CASE WHEN a.ORDINAL_POSITION = b.MAX_POS THEN ' ORDER BY REFERENCE_TYPE' ELSE ' UNION ' END
FROM INFORMATION_SCHEMA.COLUMNS a CROSS JOIN
(SELECT MAX(ORDINAL_POSITION) AS MAX_POS
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME='NONCONFORMANCE'
AND (COLUMN_NAME like '%REFERENCE[_]%' OR
COLUMN_NAME LIKE '%DATE[_]%' OR
COLUMN_NAME LIKE '%NUM[_]%')) b CROSS JOIN
(SELECT COLUMN_NAME AS PK, ROW_NUMBER() OVER(ORDER BY ORDINAL_POSITION) AS rn
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME='NONCONFORMANCE') c
WHERE a.TABLE_NAME='NONCONFORMANCE'
AND (a.COLUMN_NAME like '%REFERENCE[_]%' OR
a.COLUMN_NAME LIKE '%DATE[_]%' OR
a.COLUMN_NAME LIKE '%NUM[_]%')
AND c.rn=1
This is the result of the query (it will be different for every table):
SELECT 'NCM_REFERENCE_1' AS REFERENCE_TYPE, NCM_REFERENCE_1 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_1 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_2' AS REFERENCE_TYPE, NCM_REFERENCE_2 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_2 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_3' AS REFERENCE_TYPE, NCM_REFERENCE_3 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_3 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_4' AS REFERENCE_TYPE, NCM_REFERENCE_4 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_4 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_5' AS REFERENCE_TYPE, NCM_REFERENCE_5 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_5 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_6' AS REFERENCE_TYPE, NCM_REFERENCE_6 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_6 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_7' AS REFERENCE_TYPE, NCM_REFERENCE_7 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_7 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_8' AS REFERENCE_TYPE, NCM_REFERENCE_8 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_8 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_9' AS REFERENCE_TYPE, NCM_REFERENCE_9 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_9 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_10' AS REFERENCE_TYPE, NCM_REFERENCE_10 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_10 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_11' AS REFERENCE_TYPE, NCM_REFERENCE_11 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_11 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_12' AS REFERENCE_TYPE, NCM_REFERENCE_12 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_12 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_13' AS REFERENCE_TYPE, NCM_REFERENCE_13 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_13 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_14' AS REFERENCE_TYPE, NCM_REFERENCE_14 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_14 IS NOT NULL UNION
SELECT 'NCM_REFERENCE_15' AS REFERENCE_TYPE, NCM_REFERENCE_15 AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_REFERENCE_15 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_1' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_1) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_1 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_2' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_2) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_2 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_3' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_3) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_3 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_4' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_4) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_4 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_5' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_5) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_5 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_6' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_6) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_6 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_7' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_7) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_7 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_8' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_8) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_8 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_9' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_9) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_9 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_10' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_10) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_10 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_11' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_11) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_11 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_12' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_12) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_12 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_13' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_13) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_13 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_14' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_14) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_14 IS NOT NULL UNION
SELECT 'NCM_NUM_REFERENCE_15' AS REFERENCE_TYPE, CONVERT(varchar,NCM_NUM_REFERENCE_15) AS textValue, b.SYSID AS XREF_SYSID FROM NONCONFORMANCE a INNER JOIN XREF_TABLE b ON a.NONCONFORMANCE_SYSID = b.PK AND b.TABLE_NAME = 'NONCONFORMANCE' WHERE NCM_NUM_REFERENCE_15 IS NOT NULL ORDER BY REFERENCE_TYPE