/* 

   ChemShell-Parallel API

   you.lu@stfc.ac.uk
   thomas.keal@stfc.ac.uk

   cf chemsh/masterslave.c in Tcl-ChemShell

   Originally designed for ChemShell but eventually developed general-purpose
   "chemsh_parallel_" is a temporary prefix and will be renamed later

*/

/* YL 29/11/2017: renamed the terminology to "master/replica" due to appropriateness
   see: https://en.wikipedia.org/wiki/Master/slave_(technology)
*/

/* Copyright (C) 2017 The authors of Py-ChemShell

   This file is part of Py-ChemShell.

   Py-ChemShell is free software: you can redistribute it and/or modify
   it under the terms of the GNU Lesser General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

   Py-ChemShell is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with Py-ChemShell.  If not, see
   <http://www.gnu.org/licenses/>.  */


//#define _GNU_SOURCE
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

// file system I/O
#include <sys/stat.h>
#include <unistd.h>

// debugging
#include <errno.h>


#define PAR_END -1
#define PAR_ENV -2
#define PAR_SYS -3
#define PAR_MEM -4
// YL 18/01/2023: increased the length to allow the trailing \0 and avoid warning: ‘__builtin___sprintf_chk’ writing a terminating nul past the end of the destination [-Wformat-overflow=]
#define BASELEN  9


/* Store pointers to parallel function calls */
struct Chemsh_Parallel_C_Function_Struct {
  int handle;
  int (*func)();
  struct Chemsh_Parallel_C_Function_Struct *next;
};

typedef struct Chemsh_Parallel_C_Function_Struct *functionType;

static functionType pf_first, pf_last;
static int handle_last;
// length of WG basename: _wg_0123

// declare Fortran subroutines/functions
int chemsh_parallel_f_initialise(int *, int *, int *, int *);
int chemsh_parallel_f_finalise(void);
int chemsh_parallel_f_synchronise(void);
int chemsh_parallel_f_broadcast(int *, int *, int *, int *);
int chemsh_parallel_f_get_nprocs(void);
int chemsh_parallel_f_get_nwgs(void);
int chemsh_parallel_f_get_nnodes_wg(void);
int chemsh_parallel_f_get_nnodes_subset(void);
int chemsh_parallel_f_get_wg(void);
int chemsh_parallel_f_get_node(void);

static int debug = 0;


// ### PRIVATE METHODS ###

// get workgroup diretory name
const char *chemsh_parallel_c_get_wg_dirname() {

    char *wg_dir_name = malloc(BASELEN);

    sprintf(wg_dir_name, "_wg_%04d", chemsh_parallel_f_get_wg());

    return wg_dir_name;
}


// get output filename of the workgroup
const char *chemsh_parallel_c_get_wg_out_filename() {

    char *wg_out_name = malloc(BASELEN+4);
    const char *chemsh_parallel_c_get_wg_dirname();

    sprintf(wg_out_name, "%s.log", chemsh_parallel_c_get_wg_dirname());

    return wg_out_name;

}


// get error filename of the workgroup
const char *chemsh_parallel_c_get_wg_err_filename() {

    char *wg_err_name = malloc(BASELEN+4);
    const char *chemsh_parallel_c_get_wg_dirname();

    sprintf(wg_err_name, "%s.err", chemsh_parallel_c_get_wg_dirname());

    return wg_err_name;

}


// get archive dirname of the workgroup
const char *chemsh_parallel_c_get_wg_archive_dirname() {

    static char wg_archive_dirname[17];
    const char *chemsh_parallel_c_get_wg_dirname();

    sprintf(wg_archive_dirname, "%s", chemsh_parallel_c_get_wg_dirname());
    strcat(wg_archive_dirname, ".OLD");

    return wg_archive_dirname;

}


// get archive output filename of the workgroup
const char *chemsh_parallel_c_get_wg_out_archive_name() {

    static char wg_out_archive_name[17];
    const char *chemsh_parallel_c_get_wg_out_filename();

    sprintf(wg_out_archive_name, "%s", chemsh_parallel_c_get_wg_out_filename());
    strcat(wg_out_archive_name, ".OLD");

    return wg_out_archive_name;

}


// get archive error filename of the workgroup
const char *chemsh_parallel_c_get_wg_err_archive_name() {

    static char wg_err_archive_name[17];
    const char *chemsh_parallel_c_get_wg_err_filename();

    sprintf(wg_err_archive_name, "%s", chemsh_parallel_c_get_wg_err_filename());
    strcat(wg_err_archive_name, ".OLD");

    return wg_err_archive_name;

}


int chemsh_parallel_c_register_function(int (*func)()) {
    /*
    returns an integer handle which can be used to invoke the routine
    passed as an argument on both master and replica nodes nodes of the system
 
    it is assumed that the message passing libraries of the all parallel
    routines are consistent with the message passing used here.
    */
    functionType new;
  
    new = (functionType) malloc(sizeof(struct Chemsh_Parallel_C_Function_Struct));
  
    if(!pf_first) {
        pf_first = pf_last = new;
    } else {
        pf_last->next = new;
        pf_last = new;
    }

    new->next = NULL;
    new->func = func;
    new->handle = ++handle_last;
  
//    if(debug) printf("\n >>> c_register_function: node %d pf_first %d\n", wg_nodeid(), pf_first->handle);
//    if(debug) printf("\n >>> c_register_function: node %d pf_last %d\n", wg_nodeid(), pf_last->handle);
//    if(debug) printf("\n >>> c_register_function: node %d register %d\n", wg_nodeid(), new->handle);
    
    return new->handle;

}


int chemsh_parallel_c_exec_function(int handle, int debug) {

    fflush(stdout);

    int lenbuf, itype, ifrom;
    int handle2;
    int i, ierror;
    functionType pf;

    if(debug) printf("\n >>> ChemShell Parallel module: debug level = %d\n", debug); 
 
    lenbuf = sizeof(int);
    handle2 = handle;
    itype = 101;
    ifrom = 0;

    chemsh_parallel_f_broadcast(&itype, &handle2, &lenbuf, &ifrom);

    // locate function in table
    for(pf=pf_first; pf; pf=pf->next){
        if(debug) printf("\n >>> (DEBUG) chemsh_parallel_c_exec_function: locating function in table... pf->handle=%d, pf=%d\n", pf->handle, handle);
        // if handle found, breaks the loop; otherwise `pf` will be NULL (`pf->next`)
        if(pf->handle == handle) break;
    }
  
    if(debug) printf("\n >>> (DEBUG) c chemsh_parallel_c_exec_function: start master exec\n");
  
    // execute it in the master node
    ierror = pf->func();
  
    if(debug) printf("\n >>> (DEBUG) c chemsh_parallel_c_exec_function: master code %d\n", ierror);
   
    return ierror;
}

// FIXME: not in use?
int chemsh_parallel_c_redirect_wg_output(char *wg_outname, char *wg_errname) {

    FILE *wg_stdout, *wg_stderr;

    // YL: have to truncate
    // TODO: make the constant 10 a Fortran function
    wg_outname[BASELEN] = '\0';
    wg_errname[BASELEN] = '\0';

    wg_stdout = freopen(wg_outname, "a", stdout);

    return 0;

}


void chemsh_parallel_c_print_banner() {

    if(chemsh_parallel_f_get_node() == 0 && chemsh_parallel_f_get_nprocs() > 1) {
        printf(" %s\n", "----------------------------------------------------------------------");
        if (chemsh_parallel_f_get_nwgs() == 1) {
            printf(" ChemShell parallel environment initialised\n");
            printf(" Number of MPI processes : %d\n", chemsh_parallel_f_get_nprocs());
        } else {
            printf(" ChemShell parallel environment initialised with task farming\n");
            printf(" Number of MPI processes : %d\n", chemsh_parallel_f_get_nprocs());
            printf(" Number of workgroups    : %d\n", chemsh_parallel_f_get_nwgs());
            printf(" Processes per workgroup : %d\n", chemsh_parallel_f_get_nnodes_wg());
            printf(" Processes in subset     : %d\n", chemsh_parallel_f_get_nnodes_subset());
            printf(" Current workgroup ID    : %d\n", chemsh_parallel_f_get_wg());
        }
        printf(" %s\n", "----------------------------------------------------------------------");
    }

    fflush(stdout);

}


// ### END PRIVATE METHODS ###


// ### API METHODS ###
// do NOT bring up the following functions to above the PRIVATE METHODS section, otherwise the latters need to be redeclared!

void chemsh_parallel_c_initialise(int nWorkgroups, int nProcsMM, int ga_memory, int debug) {

    int   iret, ierror;
    char *buffer = (char *) malloc(255);
    FILE *wg_stdout, *wg_stderr;

    // initialise parallel environment
    // initialise task farming: consider nWorkgroups=1 a special case
    ierror = chemsh_parallel_f_initialise(&nWorkgroups, &nProcsMM, &ga_memory, &debug);

    if(nWorkgroups > 1) {

        if(chemsh_parallel_f_get_node() == 0) {

            // we'd better archive the exsiting WG dir first but it's not so straightforward to delete a non-empty dir in C

            // mkdir
            ierror = mkdir(chemsh_parallel_c_get_wg_dirname(), (S_IRWXU | S_IRWXG | S_IRWXO));

            // sync
            chemsh_parallel_f_synchronise();

            // chdir
            ierror = chdir(chemsh_parallel_c_get_wg_dirname());
            if(ierror == -1) {
                printf("\n >>> chemsh_parallel_c_initialise: ERROR chdir %s...\n", chemsh_parallel_c_get_wg_dirname());
            }

            if(chemsh_parallel_f_get_wg() == 0) {

            //    // back up the .err files
            //    ierror = rename(chemsh_parallel_c_get_wg_err_filename(), chemsh_parallel_c_get_wg_err_archive_name());

            //    // redirect stderr
            //    wg_stderr = freopen(chemsh_parallel_c_get_wg_err_filename(), "a", stderr);
	    //    if(wg_stderr == NULL) {
            //        printf("\n >>> chemsh_parallel_c_initialise: ERROR redirecting %s...\n", chemsh_parallel_c_get_wg_err_filename());
            //    }

            } else if(chemsh_parallel_f_get_wg() > 0) {

                // back up the .log and .err files
                ierror = rename(chemsh_parallel_c_get_wg_out_filename(), chemsh_parallel_c_get_wg_out_archive_name());
                ierror = rename(chemsh_parallel_c_get_wg_err_filename(), chemsh_parallel_c_get_wg_err_archive_name());

                // sync
                chemsh_parallel_f_synchronise();

                // redirect stdout
                wg_stdout = freopen(chemsh_parallel_c_get_wg_out_filename(), "a", stdout);
	        if(wg_stdout == NULL) {
                    printf("\n >>> chemsh_parallel_c_initialise: ERROR redirecting %s...\n", chemsh_parallel_c_get_wg_out_filename());
                }

                // redirect stderr
                wg_stderr = freopen(chemsh_parallel_c_get_wg_err_filename(), "a", stderr);
	        if(wg_stderr == NULL) {
                    printf("\n >>> chemsh_parallel_c_initialise: ERROR redirecting %s...\n", chemsh_parallel_c_get_wg_err_filename());
                }

            }

        } else {

            chemsh_parallel_f_synchronise();

            ierror = chdir(chemsh_parallel_c_get_wg_dirname());
	    if(ierror == -1) {
                printf("\n >>> chemsh_parallel_c_initialise: ERROR chdir %s...\n", chemsh_parallel_c_get_wg_dirname());
            }

            if(chemsh_parallel_f_get_wg() > 0) {

		// sync
                chemsh_parallel_f_synchronise();

                // redirect stdout
		wg_stdout = freopen(chemsh_parallel_c_get_wg_out_filename(), "a", stdout);
	        if(wg_stdout == NULL) {
                    printf("\n >>> chemsh_parallel_c_initialise: ERROR redirecting %s...\n", chemsh_parallel_c_get_wg_out_filename());
                }

                // redirect stderr
	        wg_stderr = freopen(chemsh_parallel_c_get_wg_err_filename(), "a", stderr);
	        if(wg_stderr == NULL) {
                    printf("\n >>> chemsh_parallel_c_initialise: ERROR redirecting %s...\n", chemsh_parallel_c_get_wg_err_filename());
                }

            }

        }

    }

    chemsh_parallel_c_print_banner();
  
    /* Setup register of parallel function calls */
    pf_first = pf_last = NULL;
    handle_last = 0;

}

void chemsh_parallel_c_finalise(void) {

  int ierror;

  ierror = chemsh_parallel_f_finalise();

}

void chemsh_parallel_c_run_replicas(void) {
  
    int code;
    int lenbuf, ifrom, itype;
    int ierr;
    int chemsh_parallel_f_get_node();
    functionType pf;
  
    while(1) {

        ifrom  =   0;
        itype  = 101;
        lenbuf = sizeof(int);

        // receive integer code
        // old way: cast the address of "code" as a pointer to a char, which is useful for reading single bytes at the address of "code", since char is a byte in size
        ierr = chemsh_parallel_f_broadcast(&itype, &code, &lenbuf, &ifrom);

        if(ierr != 0) {
            printf("\n >>> chemsh_parallel_run_replicas: ERROR calling chemsh_parallel_f_broadcast...\n");
        }

        // PAR_MEM = -4 PAR_ENV = -2 PAR_SYS = -3
        switch(code) {
    
            case PAR_MEM:
                /* TODO */
                break;
        
            case PAR_ENV:
                /* TODO */
                break;
        
            case PAR_SYS:
                break;
        
            case PAR_END:
                /* request to close down master-replica mode */
                return;
                break;
        
            default:
                /* a handle for a parallel function execution */
        
                /* locate function in table */
                for(pf=pf_first; pf; pf=pf->next){
                    if(pf->handle == code) break;
                }
        
                if (!pf){
                    fprintf(stderr, "\n >>> chemsh_parallel_run_replicas: node %d, bad parallel function handle  %d\n", chemsh_parallel_f_get_node(), code);
                    ierr = -999;
                } else {
                    /* execute function */
                    ierr = pf->func();
                }
        
                /* TODO send back return code */
                break;
     
        }
    }
}

void chemsh_parallel_c_end_replicas(void) {

    int lenbuf, itype, ifrom, code;

    /* send signal to close down replicas */
    ifrom = 0;
    lenbuf = sizeof(int);
    itype = 101;
    code = PAR_END;
    chemsh_parallel_f_broadcast(&itype, &code, &lenbuf, &ifrom);

    return;
}


int chemsh_parallel_c_get_node() {
    return chemsh_parallel_f_get_node();
}


int chemsh_parallel_c_get_nwgs() {
    return chemsh_parallel_f_get_nwgs();
}


int chemsh_parallel_c_get_nnodes_wg() {
    return chemsh_parallel_f_get_nnodes_wg();
}


// workgroup ID
int chemsh_parallel_c_get_wg() {
    return chemsh_parallel_f_get_wg();
}


int chemsh_parallel_c_get_nnodes_subset() {
    return chemsh_parallel_f_get_nnodes_subset();
}


int chemsh_parallel_c_is_root() {

    if(chemsh_parallel_f_get_node() == 0) {
        return 1;
    } else {
        return 0;
    }

}


const char *chemsh_parallel_c_get_hostname() {

    char cbuff[24];
    char *hostname = malloc(24);

    cbuff[23] = '\0';
    gethostname(cbuff, 23);

    sprintf(hostname, "%23s", cbuff);

    return hostname;

}

// ### END API METOHDS ###


