Using QGIS3 Processing algorithms from standalone PyQGIS scripts (outside of GUI)

I'm writing a script which must work outside of QGIS GUI. I call some API functions from qgis.core but I'd like to use the processing plugin.

I'm able to import processing with sys.path.append() but I can't run any process. Moreover, all "native" algs are missing in QgsApplication.processingRegistry().algorithms()

So is it possible to run processing that way ? What am I missing ?

import os, sys
from qgis.core import *
QgsApplication.setPrefixPath('/usr', True)
qgs = QgsApplication([], False)
qgs.initQgis()


sys.path.append('/usr/share/qgis/python/plugins')
from processing.core.Processing import Processing
Processing.initialize()
import processing

layer1 = QgsVectorLayer('data/ROUTE_PRIMAIRE.SHP')
layer2 = QgsVectorLayer('data/ROUTE_SECONDAIRE.SHP')

processing.run('qgis:union', layer1, layer2, 'test.shp') # returns nothing

I'm on QGIS 3.0.1 - Debian 9

Answer

You can run a QGIS Processing algorithm in standalone (no GUI) mode in this way:

import sys

from qgis.core import (
     QgsApplication, 
     QgsProcessingFeedback, 
     QgsVectorLayer

)

# See https://gis.stackexchange.com/a/155852/4972 for details about the prefix 
QgsApplication.setPrefixPath('/usr', True)
qgs = QgsApplication([], False)
qgs.initQgis()

# Append the path where processing plugin can be found
sys.path.append('/docs/dev/qgis/build/output/python/plugins')


import processing
from processing.core.Processing import Processing
Processing.initialize()

layer1 = QgsVectorLayer('/path/to/geodata/lines_1.shp', 'layer 1', 'ogr')
layer2 = QgsVectorLayer('/path/to/geodata/lines_2.shp', 'layer 2', 'ogr')

# You can see what parameters are needed by the algorithm  
# using: processing.algorithmHelp("qgis:union")
params = { 

    'INPUT' : layer1,
    'OVERLAY' : layer2, 
    'OUTPUT' : '/path/to/output_layer.gpkg|layername=output'
}
feedback = QgsProcessingFeedback()

res = processing.run('qgis:union', params, feedback=feedback)
res['OUTPUT'] # Access your output layer

Native Algorithms

Now, if you want to use a native algorithm (i.e., an algorithm from the native provider, whose algorithms are written in C++), you need to add the provider after initializing Processing:

import sys

from qgis.core import (
     QgsApplication, 
     QgsProcessingFeedback, 
     QgsVectorLayer
)
from qgis.analysis import QgsNativeAlgorithms


# See https://gis.stackexchange.com/a/155852/4972 for details about the prefix 
QgsApplication.setPrefixPath('/usr', True)
qgs = QgsApplication([], False)
qgs.initQgis()

# Append the path where processing plugin can be found
sys.path.append('/docs/dev/qgis/build/output/python/plugins')

import processing
from processing.core.Processing import Processing

Processing.initialize()
QgsApplication.processingRegistry().addProvider(QgsNativeAlgorithms())

layer = QgsVectorLayer('/path/to/geodata/lines.shp', 'my layer', 'ogr')

# You can see what parameters are needed by the algorithm  
# using: processing.algorithmHelp("native:extractvertices")
params = {
    'INPUT': layer,
    'OUTPUT': 'memory:'

}
feedback = QgsProcessingFeedback()

res = processing.run("native:extractvertices", params, feedback=feedback)
res['OUTPUT'] # Access your output layer