*! version 0.1.1 jsl 2008-01-31 - cleanup

* DO: save both methods to the same data file

/*
Names of locals used by -pvdata3-

    itemVARS        names of variables (called item variables) to be plotted
    itemN           # of items
    itemNCATS       # of categories in items
    itemMAXCAT      largest item category
    itemMINCAT      smallest item category
    itemORDERED     items ordered by autocategory

    otherVARS       other variables to include in dataset
    srcVARS         names of variables from the source dataset
    newVARS         names of variables that are created
*/

capture program drop pvdata3
program define pvdata3
    version 9.2

    local qui "qui" // change to "" if you want to see output

    syntax varlist ///
            , id(varname)               /// id variable
            [AUTOorder(integer -999)]   /// arrange items in order corresponding to percent in this category
            [NOSort]                    /// Don't sort items from first item to last item
            [DEScending) ]              /// Sort in descending order
            [SORTVars(varlist) ]        /// Sort on these vars; if not listed, use varlist
            [Method(string)]            /// uniform or stacked method for plotting
                                        ///     stacked places all paths on top of each other
                                        ///     uniform tries to make paths straight, adding white space to graph
            [OTHERVars(varlist)]        /// Other variables to keep in dataset.
                                        /// not be used often
            [SAMPLE(integer 100)]       /// sample this percent of cases (used to test program or plot large dataset)
            [DEBUG]                     /// test mode
            [OLDSORT]

//  BOOKKEEPING

    * order for output of variables
    local srcVARS "`varlist' `id' `colorvar' `othervars'"
    * new variables that are added
    local newVARS ""
    * with new data, old graph is dropped and interactive mode is off
    global pvinteractive = 0

//  METHOD USED TO ARRANGE LINES FOR PLOTTING

    * default method is stacked
    if "`method'"=="" {
        local method "stacked"
    }

//  SAMPLE CASES for testing

    if `sample'<100 {
        set seed 11
        tempvar r
        gen `r' = uniform()
        keep if `r'<.`sample'
        drop `r'
    }

//  INFORMATION ON DATA BEING PLOTTED

    * items to plot
    local itemVARS "`varlist'"
    * number of items
    local itemN : word count `itemVARS'
    * number of categories in items
    local itemNCATS = 0
    * min and max values of item categories
    local itemMAXCAT = -9999
    local itemMINCAT =  9999
    * find min and max values across all items
    forvalues i = 1(1)`itemN' {
        local tmpvar : word `i' of `itemVARS'
        * count number of categories in tabulation
        `qui' tab `tmpvar'
        local icatsK = r(r)
        if `icatsK'>`itemNCATS' {
            local itemNCATS = r(r)
        }
        * find minimum and maximum category values
        `qui' sum `tmpvar'
        local imax = r(max)
        local imin = r(min)
        if `imax'>`itemMAXCAT' {
            local itemMAXCAT = r(max)
        }
        if `imin'<`itemMINCAT' {
            local itemMINCAT = r(min)
        }
    } // end forvalues through items

    * problems with number of categories
    if `itemNCATS'<=1 {
        di in red "pv data not created since no item has more than 1 category."
        exit
    }

    * problem if cateories are not in sequence (i.e., gaps)
    local itemNCATS2 = `itemMAXCAT' - `itemMINCAT' + 1
    if `itemNCATS'!=`itemNCATS2' {
        di in red "ERROR: category values are not sequential"
        di "itemNCATS:  `itemNCATS'"
        di "itemMAXCAT: `itemMAXCAT'"
        di "itemMINCAT: `itemMINCAT'"
        tab `tmpvar'
        exit
    }

//  AUTOORDER items from largest to smallest in given category

    if `autoorder'!=-999 {
        * col 1 is item number; column 2 is size in specified category
        matrix def s = J(`itemN',2,-99)
        * get counts for each variable for given autoorder item
        local varnum = 0
        foreach var in `itemVARS' {
            local ++varnum
            * count times varaible is equal to autoorder value
            qui count if `var'==`autoorder'
            local n`varnum' = r(N)
            matrix s[`varnum',1]=`varnum'
            matrix s[`varnum',2]=`n`varnum''
        }
        * sort matrix by count of # of cases equal to autoorder
        mata: s = st_matrix("s")
        mata: s = sort(s,2)
        mata: st_matrix("s",s)
        * create list of items ordered by occurence of autoorder value
        local itemORDERED ""
        forvalues item = `itemN'(-1)1 {
            local varnum = s[`item',1]
            local varnm : word `varnum' of `varlist'
            local itemORDERED "`itemORDERED' `varnm'"
        }
        di _new "Items were reordered by size of category `autoorder'."
        di _new "Original order: `itemVARS'"
        di      "     New order: `itemORDERED'"
        * make ordered list the list used in rest of program
        local itemVARS "`itemORDERED'"
    }

//  SUMMARIZE INPUT

    if "`debug'"=="debug" {
        di _new "! local used by pvdata3"
        di _new
        di "itemVARS    : `itemVARS'"
        di "id          : `id'"
        di "othervars   : `othervars'"
        di "sortvars    : `sortvars'"
        di "descending  : `descending'"
        di "nosort      : `nosort'"
        di "sample      : `sample'"
        di "itemN       : `itemN'"
        di "itemNCATS   : `itemNCATS'"
        di "method      : `method'"
        di _new
    }

//  SORT data

    if "`nosort'"!="nosort" {
        * if sort vars not listed, sort by all items
        if "`sortvars'"=="" {
            local sortvars "`itemVARS'"
        }
        * add minus to name for descending sort
        local sortorder "ascending"
        if "`descending'"=="descending" {
            local sortorder "decending"
            local tmpsortvars ""
            foreach v in `sortvars' {
                local tmpsortvars "`tmpsortvars' -`v'"
            }
        }
        * ascending sort
        else {
            local tmpsortvars "`sortvars'"
        }
        * original sort ethod
        if "`oldsort'"=="oldsort" {
            sort `sortvars'
        }
        else {
            gsort `tmpsortvars'
        }
        di _new "Sorted `sortorder' by: `sortvars'"

    } // if sort

//  CREATE DATA TO PLOT

    * only keep variables specified in pvdata command
    keep  `id' `itemVARS' `othervars' `sortvars'
    order `id' `itemVARS' `othervars' `sortvars'

    *  _pvrownum is row number after sorting; this is the id number used
    *       for computing offsets
    `qui' gen _pvrownum = _n
    local newVARS "`newVARS' _pvrownum"
    label var _pvrownum "PV row number after sorting"
    sort _pvrownum
    label data "pv: temporaroy source data"
    `qui' save x-pvsource, replace

    * rename items to i#
    local vnum = 0
    foreach var in `itemVARS' {
        local ++vnum
        if "`debug'"=="debug" {
            di _new "! `var' = renamed as => i`vnum'"
        }
        rename `var' i`vnum'
        note i`vnum': renamed `var'.
    }

    * reshape to long from wide
    `qui' reshape long i, i(_pvrownum) j(item)

    * name variables for start of line segment
    rename i catSTy
    label var catSTy "PV start y-coord for category"
    rename item itemSTx
    label var itemSTx "PV start x-coord for item"
    label data "pv: 1st coordinate data"
    `qui' save x-pvcoord1, replace
    if "`debug'"=="debug" {
        di _new "! x-pvcoord1 variables"
        sum
        describe
    }

    * offset data by 1 row to create 2nd coordinate
    `qui' drop in 1 // drop 1st to offset by 1
    rename itemSTx itemENx
    rename catSTy  catENy
    label var catENy  "PV end y-coord for category"
    label var itemENx "PV end y-coord for item"
    label data "pv: 2nd coordinate data"
    `qui' save x-pvcoord2, replace
    `qui' use  x-pvcoord1, clear
    `qui' merge using x-pvcoord2
    if "`debug'"=="debug" {
        di _new "! merge x-pvcoord1 and x-pvcoord2"
        tab _merge
    }
    `qui' drop _merge

    * merge source data which is used to check things later on; is not plotted
    `qui' sort  _pvrownum itemSTx
    `qui' merge _pvrownum using x-pvsource

    * clean up coordinate data
    if "`debug'"=="debug" {
        di _new "! merge x-pvcoord1 and x-pvcoord2 with source data"
        tab _merge
    }
    `qui' drop _merge

    * do not need start item equal to last item
    `qui' drop if itemSTx==`itemN'

    * create variables that will hold offsets
    `qui' gen catSTyoff = .
    label var catSTyoff "catSTy offsets"
    `qui' gen catENyoff = .
    label var catENyoff "catENy offsets"

    * add sequence number for rows of pv dataset - probably not needed
    gen _pvsortorder = _n
    lab var _pvsortorder "PV sort order of expanded data"

    * order to save variables
    local newVARS "_pvsortorder catSTy catENy catSTyoff catENyoff itemSTx itemENx"

// =====================================================
//  STACKED PATHS: same algorithm used by AVL in PV interactive.

    if "`method'"=="stacked" {

        * default offsets of stacked elements
        local yoffset = 0 // change to some value to put space between grid and paths
        matrix offsetST = J(`itemNCATS',`itemN',`yoffset') // offset for start value
        matrix offsetEN = offsetST // offset for end value

        * loop through path segments
        local nobs = _N
        forvalues obs = 1(1)`nobs' {

            * get coordinates for line segment in curret row of data
            local iST = itemSTx[`obs']
            local iEN = itemENx[`obs']
            local cST =  catSTy[`obs'] - `itemMINCAT' + 1
            local cEN =  catENy[`obs'] - `itemMINCAT' + 1

            * increment y axis by 1 unit
            matrix offsetST[`cST',`iST'] = offsetST[`cST',`iST'] + 1
            matrix offsetEN[`cEN',`iEN'] = offsetEN[`cEN',`iEN'] + 1
            local offST = offsetST[`cST',`iST']
            local offEN = offsetEN[`cEN',`iEN']

            * increase y coordinate by # of offsets times size of delta
            `qui' replace catSTyoff = `offST' in `obs'
            `qui' replace catENyoff = `offEN' in `obs'

        } // end forvalues for obs

    }

// =====================================================
//  UNIFORM PATHS: paths are horizontal for given id staying in given category

    else if "`method'"=="uniform" {

        * loop over times and categories
        *   : create locals for each item/category and each category
        forvalues itemi = 1(1)`itemN' {
            forvalues catc = 1(1)`itemNCATS' {
                * y coord for item i and category c
                local y_i`item'_c`catc' = -1 // start below 0 since 1st line adds 1
                * y coord for category c ignoring item
                local y_cat`catc' = -1
            }
        } // end forvalues over items

        * loop through rows to plot
        local idprior = -999
        local nobs = _N
        forvalues obs = 1(1)`nobs' {
            * get id of current row
            local idnew = _pvrownum[`obs']

            * if this row starts a new person's data
            *  - first time for new id, check if ever in each category
            if `idnew'!=`idprior' {

                * indictor for if ever in this category
                forvalues cat = 1(1)`itemNCATS' {
                    local incat`cat' = 0
                }

                * was this category occupied by this person?
                forvalues cat = 1(1)`itemNCATS' {
                    * loop through items
                    foreach itemis in `itemVARS' {
                        * category for this item
                        local itemcat = `itemis'[`obs']
                        * is item is in this category
                        if `itemcat'==`cat' {
                            * if yes, that category is used by this person at least once
                            local incat`cat'= 1
                        }
                    } // itemis loop
                } // cat loop

                * loop through categories; if ever in that category
                *   add 1 to offset
                forvalues cat = 1(1)`itemNCATS' {
                    local y_cat`cat' = `y_cat`cat'' + `incat`cat''
                }
            } // end if idnew ne idprior; i.e., first row for this id

            * current id becomes the prior id so won't go through
            *   prior if loop
            local idprior `idnew'

            * start category for current id
            local c = catSTy[`obs']
            * add delta for this category
            `qui' replace catSTyoff = `y_cat`c'' in `obs'
            local c = catENy[`obs']
            `qui' replace catENyoff   = `y_cat`c'' in `obs'

        } // end forvalues loop through observations

    } // end of uniform plot method

//  ======================================================
//  invalid offset method

    else {
        * DO: add plot method with no offset
        di in red "ERROR: invalid method chosen"
        exit
    }

//  check data

        if "`debug'"=="debug" {
            di _new "! sample of data to plot"
            list  _pvrownum itemSTx catSTy itemENx catENy ///
                catSTyoff catENyoff in 1/50, compress
        }

//  add label for items

    * axis labels for items
    local itemlbl ""
    local itemnum = 0
    * 2008-01-30
    foreach item in `itemVARS' {
        local ++itemnum
        local itemlbl "`itemlbl' `itemnum' `item'"
    }

    label def itemlbl `itemlbl'
    label val itemSTx itemlbl
    label val itemENx itemlbl

//  reorder data

    order `newVARS' `sourcevars'

//  add characteristics to be used in pvplot3

    char define _dta[pvdata] "pvdata3"
    char define _dta[pvsortorder] "`sortorder'"
    char define _dta[pvsortvars]  "`sortvars'"
    char define _dta[pvmethod]    "`method'"
    char define _dta[pvitemvars]  "`itemVARS'"
    char define _dta[pvothervars] "`othervars'"
    char define _dta[pvitemncats] "`itemNCATS'" // number of categories
    char define _dta[pvitemmincats] "`itemMINCATS'" // minimum category value
    char define _dta[pvitemmaxcats] "`itemMAXCATS'" // maximum category value


end
exit

HISTORY

* version 0.0.5 jsl 2007-12-22 - add characteristics to the output dataset
* version 0.0.4 jsl 2007-12-22 - add uniform method for line placement
* version 0.0.3 jsl 2007-12-21 - cleanup code
* version 0.0.2 jsl 2007-12-21 - autoorder(#)
* version 0.0.1 jsl 2007-12-21 - pvdata2 becomes pvdata3 - descending sort
* version 0.0.1 jsl 2007-12-19 - pvdata1 becomes pvdata2
*  - add sortvars
* version 0.0.7 jsl 2008-01-28 - fix output order with autoorder
* version 0.0.9 jsl 2008-01-29 - add chars with names of variables
* version 0.0.8 jsl 2008-01-28 - fix sortvars bug
* version 0.1.0 jsl 2008-01-30 - autoorder bug