Pyccel stands for Python extension language using accelerators.

Dear Pyccel team, I'm working on a Windows pc, and apparently I have all the pyccel-requirements already installed. Everything is easily provided for free by:

• Visual Studio Community (2017 or 2019), with Python 3.7 64 bit [Python Software Fundation] ... (not Conga, or Anaconda or others...);
• Intel oneAPI Base Toolkit (compilers, pre-optimized libraries (MKL included), analysis tools,....)
• Intel oneAPI HPC Toolkit (C++,Fortran,OpenMP,MPI,...)

I don't know if you provide support for testing Pyccel on new platforms, just let me know and eventually close this issue. In principle, if I had info about the meaning of the error-messages, I could try to fix something...

Now: I installed Pyccel with pip and I'm running sequentially the pyccel-tests provided in the tests\run_tests.bat. I passed all the test until the following:

python -m pytest -rx --ignore=symbolic --ignore=ndarrays -v --maxfail=1 -v ../tests/epyccel -m "not parallel"

when I get the following error message (I append only the first and last lines):

==================================================================================================================== test session starts ====================================================================================================================
platform win32 -- Python 3.7.8, pytest-6.2.3, py-1.10.0, pluggy-0.13.1 -- C:\Users\myuser\AppData\Local\Programs\Python\Python37\python.exe
cachedir: .pytest_cache
rootdir: C:\Users\myuser\Documents\00_Tools\Pyccel, configfile: pytest.ini
plugins: forked-1.3.0, xdist-2.2.1
collected 2659 items / 13 deselected / 2646 selected

epyccel\test_array_as_func_args.py::test_array_int_1d_scalar_add[fortran] FAILED                                                                                                                                                                       [  0%]

========================================================================================================================= FAILURES ==========================================================================================================================
___________________________________________________________________________________________________________ test_array_int_1d_scalar_add[fortran] ___________________________________________________________________________________________________________

language = 'fortran'

    def test_array_int_1d_scalar_add(language):
        @types( 'int8[:]' , 'int8' , 'int')
        @types( 'int16[:]', 'int16', 'int')
        @types( 'int32[:]', 'int32', 'int')
        @types( 'int64[:]', 'int64', 'int')
        def array_int_1d_scalar_add( x, a, x_len ):
            for i in range(x_len):
                x[i] += a
        f1 = array_int_1d_scalar_add
>       f2 = epyccel(f1, language=language)

epyccel\test_array_as_func_args.py:24:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\Users\myuser\AppData\Local\Programs\Python\Python37\lib\site-packages\pyccel\epyccel.py:335: in epyccel
    mod, fun = epyccel_seq( python_function_or_module, **kwargs )
C:\Users\myuser\AppData\Local\Programs\Python\Python37\lib\site-packages\pyccel\epyccel.py:180: in epyccel_seq
    output_name = module_name)
C:\Users\myuser\AppData\Local\Programs\Python\Python37\lib\site-packages\pyccel\codegen\pipeline.py:458: in execute_pyccel
    verbose)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

codegen = <pyccel.codegen.codegen.Codegen object at 0x00000251FF5E8B48>, language = 'fortran', pyccel_dirpath = 'C:\\Users\\myuser\\Documents\\00_Tools\\Pyccel\\tests\\__epyccel__\\__pyccel__', compiler = 'gfortran', mpi_compiler = None
accelerator = None
dep_mods = ('C:\\Users\\myuser\\Documents\\00_Tools\\Pyccel\\tests\\__epyccel__\\__pyccel__\\bind_c_mod_a9gxdwww2u2p_a9gxdwww2u2p', 'C:\\Users\\myuser\\Documents\\00_Tools\\Pyccel\\tests\\__epyccel__\\__pyccel__\\mod_a9gxdwww2u2p_a9gxdwww2u2p')
libs = [], libdirs = [], includes = ['C:\\Users\\myuser\\Documents\\00_Tools\\Pyccel\\tests\\__epyccel__\\__pyccel__', 'C:\\Users\\myuser\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\numpy\\core\\include']
flags = ' -O3 -fPIC  -I"C:\\Users\\myuser\\Documents\\00_Tools\\Pyccel\\tests\\__epyccel__\\__pyccel__"', sharedlib_modname = 'mod_a9gxdwww2u2p_a9gxdwww2u2p', verbose = False

    def create_shared_library(codegen,
                              language,
                              pyccel_dirpath,
                              compiler,
                              mpi_compiler,
                              accelerator,
                              dep_mods,
                              libs,
                              libdirs,
                              includes='',
                              flags = '',
                              sharedlib_modname=None,
                              verbose = False):
... code lines
... code lines
... code lines
... code lines
            if verbose:
                print(out)
            if p.returncode != 0:
                err_msg = "Failed to build module"
                if verbose:
                    err_msg += "\n" + err
>               raise RuntimeError(err_msg)
E               RuntimeError: Failed to build module

C:\Users\myuser\AppData\Local\Programs\Python\Python37\lib\site-packages\pyccel\codegen\python_wrapper.py:148: RuntimeError
------------------------------------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------------------------------------

ERROR at shared library generation stage
----------------------------------------------------------------------------------------------------------------- Captured stdout teardown ------------------------------------------------------------------------------------------------------------------
Tearing down!

this message does not help me to fix the problem, should I run other basic tests before this?

many thanks and best regards

Describe the bug Hello,

I create a program in fortran and then I transformed it to python and I use Pyccel to accelerate Python functions, but when I use numpy functions my code pyccel is slow than pure fortran , and when I don't use numpy functions pyccel be faster than my fortran code .

So is there a possibility to combine the tow together (fortran code + pyccel ) ?

To Reproduce Provide code to reproduce the behavior:

from pyccel.decorators import types
@types('float[:,:]','float[:,:](order=C)','float[:,:](order=C)')
def qr_mgs(A,Q,R):
    from numpy import matmul,norm,zeros
    n,p=A.shape
    Qt=zeros((A.shape[1],A.shape[0]))
    s=zeros((Qt.shape[0],Q.shape[1]))

    for  k in range(p): 
        Q[:,k] = A[:,k]
        Qt[k,:]=A[:,k]
        s=matmul(Qt,Q)
        for  i in range(k):
            R[i,k] =s[k,i]
            Q[:,k] = Q[:,k] - R[i,k]*Q[:,i]
        R[k,k] = norm(Q[:,k])
        Q[:,k] = Q[:,k]/R[k,k]
    return 0
from pyccel.epyccel import epyccel
qr_f90 = epyccel(qr_mgs)
def qr_fp(a):
    n,p=a.shape
    q= np.zeros((n,p),order='C')
    r= np.zeros((p,p),order='C')
   # qt=np.zeros((p,n),order='C')
    qr_f90(a,q,r)
    return q,r
""""fortran version using f2py """
%load_ext fortranmagic
%%fortran --fcompiler=gnu95

subroutine  mgs(A,n,p,Q,R)
    integer(8) ,intent(in) ::n,p   
    real(8) ,intent(in) ::A(n,p)    
    real(8), intent(out) :: R(p,p)
    real(8), intent(out) ::   Q(n,p)
    integer(8):: k,i
    Q = 0.0d+00
    R = 0.0d+00
    do k = 1,p
        Q(:,k) = A(:,k)
        do i = 1,k-1
            R(i,k) = dot_product(Q(:,i),Q(:,k))
            Q(:,k) = Q(:,k) - R(i,k)*Q(:,i)
        end do
        R(k,k) = sqrt(dot_product(Q(:,k),Q(:,k)))
        Q(:,k) = Q(:,k)/R(k,k)
    end do
end subroutine  mgs

Provide the generated code, or the error message:

import numpy as np

A=np.random.rand(700,700)
q,r=qr_fp(A)
Q,R=mgs(A)
np.linalg.norm(A-q@r) =2.8754496870175957e-13
np.linalg.norm(A-Q@R) = 2.870136485142167e-13
%timeit qr_fp(A)
22.9 s ± 242 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
%timeit mgs(A)
247 ms ± 2.84 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

Expected behavior A clear and concise description of what you expected to happen.

Language Please specify which language the python code is translated to (Fortran by default)

Additional context Add any other context about the problem here.

Implement the transpose function. In fortran when used as a temporary variable the transpose function is used. When the variable is saved, the ordering is used to ensure pointers are created. In C the function is handled through unravelling or with a new function transpose_alias_assign.

The transpose function is also used to implement the ndarray property .T and method .transpose()

Related to #806 and #458

When pyccelizing a function that does a for reduction of two arrays with openmp, a segmentation fault occurs. The same code with only one array in the for reduction does not encounter this error.

As an example take a file pyccel_test.py with two functions mult_reduct1 and mult_reduct2:

def mult_reduct1(arr : 'float[:,:]', arr1 : 'float[:,:]'):
    """
    Parameters:
        2 arrays of the same size
    """
    nx = arr.shape[1]
    ny = arr.shape[0]

    #$ omp parallel private (i, j)
    #$ omp for reduction(+: arr1)
    for i in range(nx):
        j = i * i

        arr1[i,0] = arr[i,0]**2
        arr1[i,1] = arr[i,1]**2

    #$ omp end parallel

def mult_reduct2(arr : 'float[:,:]', arr1 : 'float[:,:]', arr2 : 'float[:,:]'):
    """
    Parameters:
        3 arrays of the same size
    """
    nx = arr.shape[1]
    ny = arr.shape[0]

    #$ omp parallel private (i, j)
    #$ omp for reduction(+: arr1, arr2)
    for i in range(nx):
        j = i * i

        arr1[i,0] = arr[i,0]**2
        arr1[i,1] = arr[i,1]**2

        arr2[i,0] = arr[i,0]**2
        arr2[i,1] = arr[i,1]**2


    #$ omp end parallel

which is pyccelized with the command pyccel --openmp pyccel_test.py. The pyccelization itself gives no errors.

When calling these functions in another python file with e.g.

import pyccel_test as pt

nx = 1000
ny = 1000

array = np.random.rand(nx,ny)
array1 = np.zeros(np.shape(array), dtype=float)
array2 = np.zeros(np.shape(array), dtype=float)

pt.mult_reduct1(array, array1)
print('\n results \n')
print(np.sum(array1))

pt.mult_reduct2(array, array1, array2)
print('\n results \n')
print(np.sum(array1))
print(np.sum(array2))

the output reads

 results 

674.2675057646197
Segmentation fault (core dumped)

I am using the newest pyccel version 1.5.2

Describe the bug C has no way to rename functions, nor does it have namespaces. This makes collisions much more likely when using more than one file. Furthermore there is no way to import part of a file. This makes collisions even more likely.

To fix this problem I suggest pre-pending the name of the module to each function/variable/class defined within that module in the C code. I.e: f -> mod__f

This should prevent almost all collisions. Collisions are still possible if the user names his variables using the following pattern:

NAME1__NAME2.py containing: - function : f() NAME1.py containing: - function NAME2__f()

But this seems unlikely, especially due to the double _

Test Code Test files can be found here. When working they can be activated for C in this file

Language C

Additional Information I suspect the difficulty here will be at the function call. The namespace must be used to find the Import which created the object in order to find the module it has come from and it's original name

When a function in C returns 1 scalar variable a temporary variable is created to mimic fortran. This is not necessary, to avoid that we should make _print_return in ccode.py prints directly the return of scalar/variable without assigning a temporary variable,

Describe the bug Under very specific circumstances the code will not compile as the file numpy_version.h is not created.

To Reproduce There are several steps required to reproduce this bug:

1. Compile code requiring a cwrapper using ndarrays (generates numpy_version.h)
2. Modify pyccel/stdlib/cwrapper/cwrapper_ndarrays.h
3. Recompile code

In the last step, the following happens:

1. Check to copy cwrapper info (this is the stage that generates numpy_version.h). Nothing happens as cwrapper.h and cwrapper.c exist and are up to date
2. Check to copy cwrapper_ndarray info. The code recognises that the file pyccel/stdlib/cwrapper/cwrapper_ndarrays.h needs updating so it deletes the __pyccel__/cwrapper folder (containing numpy_version.h)
3. Copy the cwrapper folder to collect the updated pyccel/stdlib/cwrapper/cwrapper_ndarrays.h file. This time the numpy_version.h is not corrected

To fix the problem we simply need to move the cwrapper_ndarray files to their own folder

Extra notes This problem could theoretically be triggered by a clock skew

Overhaul the compilation method. A class Compiler is now used to handle all compilation steps. A class CompileObj is used to store all information required to compile a given file. Default compile configurations for basic compilers (gcc, gfortran, icc, ifort) are saved in json files in pyccel/compilers these are determined thanks to the sysconfig module. Fixes #541.

After this PR, tests run much faster. The times below show the last 3 tests run on the master and ebourne_compiling branches, followed by the average times Master branch |Linux (python3.6)|MacOs|Windows| |-------------------|--------|----------| |19m37 | 16m54 | 28m34 | |18m46 | 21m13 | 24m28 | |21m13 | 19m53 | 23m54 | | | | | |19m52 | 19m20 | 25m39 |

ebourne_compiling |Linux (python3.6)|MacOs|Windows| |-------------------|--------|----------| |13m13 | 17m23 | 15m17 | |11m59 | 16m17 | 15m25 | |15m30 | 17m52 | 18m01 | | | | | |13m34 | 17m11 | 16m11 |

Supported compilers: The following compilers have been tested:

• GNU : gcc/gfortran
• intel : icc/ifort (tested by @EmilyBourne )
• PGI : pgcc/pgfortran (tested by @saidctb )
• nvidia : nvc/nvfort (tested by @rakati )

Commit Summary

• Create CompileObj to hold all information required to compile a given file
• Create Compiler to hold all information about a given compiler
• Remove cwrapper.py. The setup_X.py file is no longer required
• Use Compiler and CompileObj to simplify pipeline.py and python_wrapper.py
• Allow multiple accelerators
• Make mpi an accelerator instead of specifying the mpi_compiler
• Specify the compiler family (e.g. GNU) instead of the compiler name
• Add a wrapper_flags argument to separate the flags for the file (fortran) from the flags for the wrapper (c)
• When compiling a program, compile everything as modules then link separately. This procedure is closer to the procedure used for shared libraries
• Rename pyc_math to pyc_math_c and pyc_math_f90. This allows us to differentiate between the .o files created from the source files which means we don't need to systematically recompile the library
• Only overwrite stdlib files if they are out of date
• Only recompile stdlib files if the source files have been updated
• Add pyccel.compilers.generate_default to generate the default json files
• Lock file during pickling. Fixes #917
• Add a file describing how to use different compilers

• Create class PyccelSymbol
• Create functions symbols that return a tuple of instances of PyccelSymbol
• Change imports in all code
• Remove function allocatable_like not used anymore.
• Remove unnecessary cast to a string.
• Add magic methods __eq__ and __hash__ to Variable and DottedVariable

Fix #1124: use correct static cast when printing integers in C.

List of modifications:

• Add f suffix to float32 float literals
• Use macro INT64_C for appending suffix to int64 integer literals
• Use macros PRId64 , PRId16, PRId8 to get correct conversion specifier (%ld %lld %hd ...) for int64, int16 and int8
• Introduce new CMacro class representing a C Macro name
• Introduce new CStringExpression representing a holder for a list of CMacros and LiteralStrings

This PR is about extending the support of OpenMP by adding more Construct and Directives and their clauses:

• Cover all the constructs that are inside the documentation;
• Add clauses to the new constructs;
• Support combined constructs;
• Add tests for new pragmas;
• Update the documentation to fit new changes and fixes.

This PR serves as a starting point for discussing the developer documentation for GPU integration with Pyccel. The goal of the documentation is to help developers understand and use Pyccel effectively when building applications that run on GPUs.

We would like to focus on the following topics in the documentation:

device and kernel decorators CUDA Built-in variables

We welcome feedback and suggestions on these topics, as well as any other ideas you may have for improving the developer documentation on GPU integration with Pyccel. Please share your thoughts on how we can make the documentation more useful and user-friendly for developers working with GPUs.

Describe the bug

Passing a non-contiguous array to a function which has been translated to C gives unexpected results.

To Reproduce

Provide code to reproduce the behaviour:

import numpy as np

def f(a : 'float[:]'):
    a[:] = 2

if __name__ == '__main__':
    arr = np.ones((2,3,4))

    f(arr[0,:,3])
    print(arr)

Error details

Python output:

[[[1. 1. 1. 2.]
  [1. 1. 1. 2.]
  [1. 1. 1. 2.]]

 [[1. 1. 1. 1.]
  [1. 1. 1. 1.]
  [1. 1. 1. 1.]]]

C output:

[[[1.000000000000 1.000000000000 1.000000000000 2.000000000000]
[1.000000000000 1.000000000000 1.000000000000 1.000000000000]
[1.000000000000 1.000000000000 1.000000000000 1.000000000000]]
[[1.000000000000 1.000000000000 1.000000000000 1.000000000000]
[1.000000000000 1.000000000000 1.000000000000 1.000000000000]
[1.000000000000 1.000000000000 1.000000000000 1.000000000000]]]

The generated code for the function is:

void f(t_ndarray a)
{
    int64_t i;
    for (i = INT64_C(0); i < a.shape[INT64_C(0)]; i += INT64_C(1))
    {
        GET_ELEMENT(a, nd_double, (int64_t)i) = INT64_C(2);
    }
}

so either GET_ELEMENT is incorrectly implemented, or array_slicing is incorrectly implemented.

Language

C

Additional context

This problem may also exist for other code snippets using different structures

This commit aims to add the autogenerated sphinx documention. a build has already been performed to generate html pages. At this point, each module has its own separate page.

Fixes #1281

This issue is solving the wrong interpretation of AugAssign into AliasAssign.

import numpy as np

if __name__ == '__main__':
    a = np.ones(5)
    b = a
    b += 3

The code above, when converted to C, used to generate this:

#include "test.h"
#include <stdlib.h>
#include "ndarrays.h"
#include <stdint.h>
int main()
{
    t_ndarray a = {.shape = NULL};
    t_ndarray b = {.shape = NULL};
    a = array_create(1, (int64_t[]){INT64_C(5)}, nd_double, false);
    array_fill((double)1.0, a);
    alias_assign(&b, a);
    free_pointer(b);
    b = INT64_C(3);
    free_array(a);
    free_pointer(b);
    return 0;
}

Now it generates this:

#include "test.h"
#include <stdlib.h>
#include "ndarrays.h"
#include <stdint.h>
#include <stdio.h>
#include <inttypes.h>
int main()
{
    t_ndarray a = {.shape = NULL};
    t_ndarray b = {.shape = NULL};
    int64_t i;
    int64_t i_0001;
    a = array_create(1, (int64_t[]){INT64_C(5)}, nd_double, false);
    array_fill((double)1.0, a);
    alias_assign(&b, a);
    for (i = INT64_C(0); i < b.shape[INT64_C(0)]; i += INT64_C(1))
    {
        GET_ELEMENT(b, nd_double, (int64_t)i) += INT64_C(3);
    }
    free_array(a);
    free_pointer(b);
    return 0;
}

which removes an unnecessary de-allocation of b after the call to alias_assign(&b, a) and replaces b = INT64_C(3) with for loop to broadcast += operation on all elements of b, which point on a.

For Fortran:

program prog_prog_test

  use test_0001

  use, intrinsic :: ISO_C_Binding, only : f64 => C_DOUBLE , i64 => &
        C_INT64_T
  implicit none

  real(f64), allocatable, target :: a(:)
  real(f64), pointer :: b(:)

  allocate(a(0:4_i64))
  a = 1.0_f64
  b(0:) => a
  b(0:) => 3_i64
  if (allocated(a)) then
    deallocate(a)
  end if

end program prog_prog_test

the line b(0:) => 3_i64 is replaced with b = b + 3_i64

Add developer docs for the syntactic stage. As per the docs, fatal errors should be used sparingly to give the user as much information as possible. This was not the case, so this PR also reduces the severity of most errors issued by the syntactic stage